{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Description:\n",
    "这里我们尝试建立一个PNN网络来完成一个ctr预测的问题。 关于Pytorch的建模流程， 主要有四步：\n",
    "1. 准备数据\n",
    "2. 建立模型\n",
    "3. 训练模型\n",
    "4. 使用和保存\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:50:43.896210Z",
     "start_time": "2020-10-21T01:50:39.097534Z"
    }
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset, TensorDataset\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchkeras import summary, Model\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:50:43.942087Z",
     "start_time": "2020-10-21T01:50:43.898206Z"
    }
   },
   "outputs": [],
   "source": [
    "# 导入数据， 数据已经处理好了 preprocess/下\n",
    "train_set = pd.read_csv('preprocessed_data/train_set.csv')\n",
    "val_set = pd.read_csv('preprocessed_data/val_set.csv')\n",
    "test_set = pd.read_csv('preprocessed_data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:50:43.986968Z",
     "start_time": "2020-10-21T01:50:43.944083Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>I3</th>\n",
       "      <th>I4</th>\n",
       "      <th>I5</th>\n",
       "      <th>I6</th>\n",
       "      <th>I7</th>\n",
       "      <th>I8</th>\n",
       "      <th>I9</th>\n",
       "      <th>I10</th>\n",
       "      <th>...</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>C22</th>\n",
       "      <th>C23</th>\n",
       "      <th>C24</th>\n",
       "      <th>C25</th>\n",
       "      <th>C26</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000381</td>\n",
       "      <td>0.000473</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009449</td>\n",
       "      <td>0.082147</td>\n",
       "      <td>0.004825</td>\n",
       "      <td>0.003656</td>\n",
       "      <td>0.040447</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>166</td>\n",
       "      <td>116</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>27</td>\n",
       "      <td>327</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.075768</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000710</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>145</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1166</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>469</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000381</td>\n",
       "      <td>0.000236</td>\n",
       "      <td>0.137931</td>\n",
       "      <td>0.004804</td>\n",
       "      <td>0.030185</td>\n",
       "      <td>0.007841</td>\n",
       "      <td>0.062157</td>\n",
       "      <td>0.024126</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>99</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>588</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>434</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011696</td>\n",
       "      <td>0.000473</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.002180</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032907</td>\n",
       "      <td>0.003193</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>188</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>862</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>27</td>\n",
       "      <td>380</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004450</td>\n",
       "      <td>0.001064</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.006119</td>\n",
       "      <td>0.039457</td>\n",
       "      <td>0.001206</td>\n",
       "      <td>0.009141</td>\n",
       "      <td>0.000887</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>348</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>296</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    I1        I2        I3        I4        I5        I6        I7        I8  \\\n",
       "0  0.0  0.000381  0.000473  0.000000  0.009449  0.082147  0.004825  0.003656   \n",
       "1  0.0  0.000127  0.000000  0.000000  0.075768  0.000000  0.000000  0.000000   \n",
       "2  0.0  0.000381  0.000236  0.137931  0.004804  0.030185  0.007841  0.062157   \n",
       "3  0.0  0.011696  0.000473  0.034483  0.002180  0.000000  0.000000  0.032907   \n",
       "4  0.0  0.004450  0.001064  0.034483  0.006119  0.039457  0.001206  0.009141   \n",
       "\n",
       "         I9  I10  ...  C18  C19  C20   C21  C22  C23  C24  C25  C26  Label  \n",
       "0  0.040447  0.0  ...  166  116    2    14    0    0   60   27  327      0  \n",
       "1  0.000710  0.0  ...  145    0    0  1166    0    1  469    0    0      0  \n",
       "2  0.024126  0.0  ...   99    0    0   588    0   11  434    0    0      0  \n",
       "3  0.003193  0.0  ...  188   34    1   862    0    0   67   27  380      1  \n",
       "4  0.000887  0.0  ...  348    0    0   296    0    9   23    0    0      0  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:50:44.017928Z",
     "start_time": "2020-10-21T01:50:43.988963Z"
    }
   },
   "outputs": [],
   "source": [
    "# 这里需要把特征分成数值型和离散型， 因为后面的模型里面离散型的特征需要embedding， 而数值型的特征直接进入了stacking层， 处理方式会不一样\n",
    "data_df = pd.concat((train_set, val_set, test_set))\n",
    "\n",
    "dense_feas = ['I'+str(i) for i in range(1, 14)]\n",
    "sparse_feas = ['C'+str(i) for i in range(1, 27)]\n",
    "\n",
    "# 定义一个稀疏特征的embedding映射， 字典{key: value}, key表示每个稀疏特征， value表示数据集data_df对应列的不同取值个数， 作为embedding输入维度\n",
    "sparse_feas_map = {}\n",
    "for key in sparse_feas:\n",
    "    sparse_feas_map[key] = data_df[key].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:50:44.032845Z",
     "start_time": "2020-10-21T01:50:44.019880Z"
    }
   },
   "outputs": [],
   "source": [
    "feature_info = [dense_feas, sparse_feas, sparse_feas_map]  # 这里把特征信息进行封装， 建立模型的时候作为参数传入"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:50:44.078722Z",
     "start_time": "2020-10-21T01:50:44.034839Z"
    }
   },
   "outputs": [],
   "source": [
    "# 把数据构建成数据管道\n",
    "dl_train_dataset = TensorDataset(torch.tensor(train_set.drop(columns='Label').values).float(), torch.tensor(train_set['Label']).float())\n",
    "dl_val_dataset = TensorDataset(torch.tensor(val_set.drop(columns='Label').values).float(), torch.tensor(val_set['Label']).float())\n",
    "\n",
    "dl_train = DataLoader(dl_train_dataset, shuffle=True, batch_size=16)\n",
    "dl_vaild = DataLoader(dl_val_dataset, shuffle=True, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T02:59:38.931537Z",
     "start_time": "2020-10-21T02:59:38.918531Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 39]) tensor([0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 0., 1.])\n"
     ]
    }
   ],
   "source": [
    "# 查看一下数据\n",
    "for b in iter(dl_train):\n",
    "    print(b[0].shape, b[1])\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立模型\n",
    "建立模型有三种方式：\n",
    "1. 继承nn.Module基类构建自定义模型\n",
    "2. nn.Sequential按层顺序构建模型\n",
    "3. 继承nn.Module基类构建模型， 并辅助应用模型容器进行封装\n",
    "\n",
    "这里我们依然会使用第三种方式， 因为embedding依然是很多层。 模型的结构如下：\n",
    "\n",
    "![](img/pnn.png)\n",
    "\n",
    "这里简单的分析一下这个模型， 说几个比较重要的细节：\n",
    "1. 这里的输入， 由于都进行了embedding， 所以这里应该是类别型的特征， 关于数值型的特征， 在把类别都交叉完了之后， 才把数值型的特征加入进去\n",
    "2. 交叉层这里， 左边和右边其实用的同样的一层， 有着同样的神经单元个数， 只不过这里进行计算的时候， 得分开算，左边的是单个特征的线性组合lz， 而右边是两两特征进行交叉后特征的线性组合lp。 得到这两个之后， 两者进行相加得到最终的组合， 然后再relu激活才是交叉层的输出。\n",
    "3. 交叉层这里图上给出的是**一个神经元**内部的计算情况， 注意这里是一个神经元内部的计算， 这些圈不是表示多个神经元。\n",
    "\n",
    "下面说一下代码的逻辑：\n",
    "1. 首先， 我们定义一个DNN神经网络， 这个也就是上面图片里面的交叉层上面的那一部分结构， 也就是很多个全连接层的一个网络， 之所以单独定义这样的一个网络， 是因为更加的灵活， 加多少层， 每层神经元个数是多少我们就可以自己指定了， 这里会涉及到一个小操作技巧。\n",
    "2. 然后就是定义整个PNN网络， 核心部分就是在前向传播。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:51:15.361422Z",
     "start_time": "2020-10-21T01:51:15.352445Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(256, 128), (128, 64)]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# python生成元素对测试， 这个可以帮助我们定义一个列表的全连接层\n",
    "a = [256, 128, 64]\n",
    "list(zip(a[:-1], a[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:51:16.466222Z",
     "start_time": "2020-10-21T01:51:16.455252Z"
    }
   },
   "outputs": [],
   "source": [
    "# 定义一个全连接层的神经网络\n",
    "class DNN(nn.Module):\n",
    "    \n",
    "    def __init__(self, hidden_units, dropout=0.):\n",
    "        \"\"\"\n",
    "        hidden_units:列表， 每个元素表示每一层的神经单元个数，比如[256, 128, 64]，两层网络， 第一层神经单元128个，第二层64，注意第一个是输入维度\n",
    "        dropout: 失活率\n",
    "        \"\"\"\n",
    "        super(DNN, self).__init__()\n",
    "        \n",
    "        # 下面创建深层网络的代码 由于Pytorch的nn.Linear需要的输入是(输入特征数量， 输出特征数量)格式， 所以我们传入hidden_units， \n",
    "        # 必须产生元素对的形式才能定义具体的线性层， 且Pytorch中线性层只有线性层， 不带激活函数。 这个要与tf里面的Dense区分开。\n",
    "        self.dnn_network = nn.ModuleList([nn.Linear(layer[0], layer[1]) for layer in list(zip(hidden_units[:-1], hidden_units[1:]))])\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "    \n",
    "    # 前向传播中， 需要遍历dnn_network， 不要忘了加激活函数\n",
    "    def forward(self, x):\n",
    "        \n",
    "        for linear in self.dnn_network:\n",
    "            x = linear(x)\n",
    "            x = F.relu(x)\n",
    "        \n",
    "        x = self.dropout(x)\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:51:20.196742Z",
     "start_time": "2020-10-21T01:51:20.188765Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Linear-1                    [-1, 8]             136\n",
      "            Linear-2                    [-1, 4]              36\n",
      "            Linear-3                    [-1, 2]              10\n",
      "            Linear-4                    [-1, 1]               3\n",
      "           Dropout-5                    [-1, 1]               0\n",
      "================================================================\n",
      "Total params: 185\n",
      "Trainable params: 185\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.000061\n",
      "Forward/backward pass size (MB): 0.000122\n",
      "Params size (MB): 0.000706\n",
      "Estimated Total Size (MB): 0.000889\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# 测试一下这个网络\n",
    "hidden_units = [16, 8, 4, 2, 1]        # 层数和每一层神经单元个数， 由我们自己定义了\n",
    "dnn = DNN(hidden_units)\n",
    "summary(dnn, input_shape=(16,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:51:22.609623Z",
     "start_time": "2020-10-21T01:51:22.593666Z"
    }
   },
   "outputs": [],
   "source": [
    "class ProductLayer(nn.Module):\n",
    "    \n",
    "    def __init__(self, mode, embed_dim, field_num, hidden_units):\n",
    "        \n",
    "        super(ProductLayer, self).__init__()\n",
    "        self.mode = mode\n",
    "        # product层， 由于交叉这里分为两部分， 一部分是单独的特征运算， 也就是上面结构的z部分， 一个是两两交叉， p部分， 而p部分还分为了内积交叉和外积交叉\n",
    "        # 所以， 这里需要自己定义参数张量进行计算\n",
    "        # z部分的w， 这里的神经单元个数是hidden_units[0], 上面我们说过， 全连接层的第一层神经单元个数是hidden_units[1]， 而0层是输入层的神经\n",
    "        # 单元个数， 正好是product层的输出层  关于维度， 这个可以看在博客中的分析\n",
    "        self.w_z = nn.Parameter(torch.rand([field_num, embed_dim, hidden_units[0]]))\n",
    "        \n",
    "        # p部分, 分内积和外积两种操作\n",
    "        if mode == 'in':\n",
    "            self.w_p = nn.Parameter(torch.rand([field_num, field_num, hidden_units[0]]))\n",
    "        else:\n",
    "            self.w_p = nn.Parameter(torch.rand([embed_dim, embed_dim, hidden_units[0]]))\n",
    "        \n",
    "        self.l_b = torch.rand([hidden_units[0], ], requires_grad=True)\n",
    "    \n",
    "    def forward(self, z, sparse_embeds):\n",
    "        # lz部分\n",
    "        l_z = torch.mm(z.reshape(z.shape[0], -1), self.w_z.permute((2, 0, 1)).reshape(self.w_z.shape[2], -1).T)# (None, hidden_units[0])\n",
    "        \n",
    "        # lp 部分\n",
    "        if self.mode == 'in':  # in模式  内积操作  p就是两两embedding先内积得到的[field_dim, field_dim]的矩阵\n",
    "            p = torch.matmul(sparse_embeds, sparse_embeds.permute((0, 2, 1)))  # [None, field_num, field_num]\n",
    "        else:  # 外积模式  这里的p矩阵是两两embedding先外积得到n*n个[embed_dim, embed_dim]的矩阵， 然后对应位置求和得到最终的1个[embed_dim, embed_dim]的矩阵\n",
    "            # 所以这里实现的时候， 可以先把sparse_embeds矩阵在field_num方向上先求和， 然后再外积\n",
    "            f_sum = torch.unsqueeze(torch.sum(sparse_embeds, dim=1), dim=1)  # [None, 1, embed_dim]\n",
    "            p = torch.matmul(f_sum.permute((0, 2,1)), f_sum)     # [None, embed_dim, embed_dim]\n",
    "        \n",
    "        l_p = torch.mm(p.reshape(p.shape[0], -1), self.w_p.permute((2, 0, 1)).reshape(self.w_p.shape[2], -1).T)  # [None, hidden_units[0]]\n",
    "        \n",
    "        output = l_p + l_z + self.l_b\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:51:25.238070Z",
     "start_time": "2020-10-21T01:51:25.213138Z"
    }
   },
   "outputs": [],
   "source": [
    "# 下面我们定义真正的PNN网络\n",
    "# 这里的逻辑是底层输入（类别型特征) -> embedding层 -> product 层 -> DNN -> 输出\n",
    "class PNN(nn.Module):\n",
    "    \n",
    "    def __init__(self, feature_info, hidden_units, mode='in', dnn_dropout=0., embed_dim=10, outdim=1):\n",
    "        \"\"\"\n",
    "        DeepCrossing：\n",
    "            feature_info: 特征信息（数值特征， 类别特征， 类别特征embedding映射)\n",
    "            hidden_units: 列表， 全连接层的每一层神经单元个数， 这里注意一下， 第一层神经单元个数实际上是hidden_units[1]， 因为hidden_units[0]是输入层\n",
    "            dropout: Dropout层的失活比例\n",
    "            embed_dim: embedding的维度m\n",
    "            outdim: 网络的输出维度\n",
    "        \"\"\"\n",
    "        super(PNN, self).__init__()\n",
    "        self.dense_feas, self.sparse_feas, self.sparse_feas_map = feature_info\n",
    "        self.field_num = len(self.sparse_feas)\n",
    "        self.dense_num = len(self.dense_feas)\n",
    "        self.mode = mode\n",
    "        self.embed_dim = embed_dim\n",
    "        \n",
    "         # embedding层， 这里需要一个列表的形式， 因为每个类别特征都需要embedding\n",
    "        self.embed_layers = nn.ModuleDict({\n",
    "            'embed_' + str(key): nn.Embedding(num_embeddings=val, embedding_dim=self.embed_dim)\n",
    "            for key, val in self.sparse_feas_map.items()\n",
    "        })\n",
    "        \n",
    "        # Product层\n",
    "        self.product = ProductLayer(mode, embed_dim, self.field_num, hidden_units)\n",
    "        \n",
    "        # dnn 层\n",
    "        hidden_units[0] += self.dense_num\n",
    "        self.dnn_network = DNN(hidden_units, dnn_dropout)\n",
    "        self.dense_final = nn.Linear(hidden_units[-1], 1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        dense_inputs, sparse_inputs = x[:, :13], x[:, 13:]   # 数值型和类别型数据分开\n",
    "        sparse_inputs = sparse_inputs.long()      # 需要转成长张量， 这个是embedding的输入要求格式\n",
    "        sparse_embeds = [self.embed_layers['embed_'+key](sparse_inputs[:, i]) for key, i in zip(self.sparse_feas_map.keys(), range(sparse_inputs.shape[1]))]   \n",
    "        # 上面这个sparse_embeds的维度是 [field_num, None, embed_dim] \n",
    "        sparse_embeds = torch.stack(sparse_embeds)\n",
    "        sparse_embeds = sparse_embeds.permute((1, 0, 2))   # [None, field_num, embed_dim]  注意此时空间不连续， 下面改变形状不能用view，用reshape\n",
    "        z = sparse_embeds\n",
    "        \n",
    "        # product layer\n",
    "        sparse_inputs = self.product(z, sparse_embeds)\n",
    "        \n",
    "        # 把上面的连起来， 注意此时要加上数值特征\n",
    "        l1 = F.relu(torch.cat([sparse_inputs, dense_inputs], axis=-1))\n",
    "        # dnn_network\n",
    "        dnn_x = self.dnn_network(l1)\n",
    "        \n",
    "        outputs = F.sigmoid(self.dense_final(dnn_x))\n",
    "        \n",
    "        return outputs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T01:51:29.839945Z",
     "start_time": "2020-10-21T01:51:29.820995Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "         Embedding-1                   [-1, 10]             790\n",
      "         Embedding-2                   [-1, 10]           2,520\n",
      "         Embedding-3                   [-1, 10]          12,930\n",
      "         Embedding-4                   [-1, 10]          10,430\n",
      "         Embedding-5                   [-1, 10]             300\n",
      "         Embedding-6                   [-1, 10]              70\n",
      "         Embedding-7                   [-1, 10]          11,640\n",
      "         Embedding-8                   [-1, 10]             390\n",
      "         Embedding-9                   [-1, 10]              20\n",
      "        Embedding-10                   [-1, 10]           9,080\n",
      "        Embedding-11                   [-1, 10]           9,260\n",
      "        Embedding-12                   [-1, 10]          12,390\n",
      "        Embedding-13                   [-1, 10]           8,240\n",
      "        Embedding-14                   [-1, 10]             200\n",
      "        Embedding-15                   [-1, 10]           8,190\n",
      "        Embedding-16                   [-1, 10]          11,590\n",
      "        Embedding-17                   [-1, 10]              90\n",
      "        Embedding-18                   [-1, 10]           5,340\n",
      "        Embedding-19                   [-1, 10]           2,010\n",
      "        Embedding-20                   [-1, 10]              40\n",
      "        Embedding-21                   [-1, 10]          12,040\n",
      "        Embedding-22                   [-1, 10]              70\n",
      "        Embedding-23                   [-1, 10]             120\n",
      "        Embedding-24                   [-1, 10]           7,290\n",
      "        Embedding-25                   [-1, 10]             330\n",
      "        Embedding-26                   [-1, 10]           5,540\n",
      "     ProductLayer-27                  [-1, 256]         239,616\n",
      "           Linear-28                  [-1, 128]          34,560\n",
      "           Linear-29                   [-1, 64]           8,256\n",
      "          Dropout-30                   [-1, 64]               0\n",
      "           Linear-31                    [-1, 1]              65\n",
      "================================================================\n",
      "Total params: 413,407\n",
      "Trainable params: 413,407\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.000153\n",
      "Forward/backward pass size (MB): 0.005898\n",
      "Params size (MB): 1.577023\n",
      "Estimated Total Size (MB): 1.583073\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "hidden_units = [256, 128, 64]\n",
    "hidden_units_copy = hidden_units.copy()\n",
    "net = PNN(feature_info, hidden_units, mode='in')\n",
    "summary(net, input_shape=(train_set.shape[1],))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T02:00:59.328537Z",
     "start_time": "2020-10-21T02:00:59.308590Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 39]) torch.Size([16])\n",
      "tensor([[0.2222],\n",
      "        [0.4534],\n",
      "        [0.6474],\n",
      "        [0.7138],\n",
      "        [0.9506],\n",
      "        [0.7158],\n",
      "        [0.8951],\n",
      "        [0.2883],\n",
      "        [0.6496],\n",
      "        [0.4923],\n",
      "        [0.7633],\n",
      "        [0.4571],\n",
      "        [0.9584],\n",
      "        [0.5374],\n",
      "        [0.9408],\n",
      "        [0.5651]], grad_fn=<SigmoidBackward>)\n"
     ]
    }
   ],
   "source": [
    "# 测试一下模型\n",
    "for fea, label in iter(dl_train):\n",
    "    print(fea.shape, label.shape)\n",
    "    out = net(fea)\n",
    "    print(out)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型的训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:05:44.010309Z",
     "start_time": "2020-10-18T07:05:43.996346Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的相关设置\n",
    "def auc(y_pred, y_true):\n",
    "    pred = y_pred.data\n",
    "    y = y_true.data\n",
    "    return roc_auc_score(y, pred)     # 计算AUC， 但要注意如果y只有一个类别的时候， 会报错\n",
    "\n",
    "loss_func = nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(params=net.parameters(), lr=0.0001)\n",
    "metric_func = auc\n",
    "metric_name = 'auc'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:05:53.386411Z",
     "start_time": "2020-10-18T07:05:44.012303Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start Training...\n",
      "========================================================================2020-10-18 15:05:44\n",
      "[step = 10] loss: 1.208, auc: 0.455\n",
      "[step = 20] loss: 1.029, auc: 0.487\n",
      "[step = 30] loss: 0.968, auc: 0.502\n",
      "[step = 40] loss: 0.925, auc: 0.498\n",
      "[step = 50] loss: 0.911, auc: 0.515\n",
      "[step = 60] loss: 0.870, auc: 0.522\n",
      "[step = 70] loss: 0.904, auc: 0.504\n",
      "[step = 80] loss: 0.873, auc: 0.520\n",
      "[step = 90] loss: 0.857, auc: 0.510\n",
      "\n",
      "EPOCH = 1, loss = 0.857,auc  = 0.510, val_loss = 0.656, val_auc = 0.420\n",
      "\n",
      "================================================================================2020-10-18 15:05:46\n",
      "[step = 10] loss: 0.588, auc: 0.615\n",
      "[step = 20] loss: 0.584, auc: 0.581\n",
      "[step = 30] loss: 0.607, auc: 0.578\n",
      "[step = 40] loss: 0.614, auc: 0.579\n",
      "[step = 50] loss: 0.628, auc: 0.572\n",
      "[step = 60] loss: 0.634, auc: 0.574\n",
      "[step = 70] loss: 0.650, auc: 0.569\n",
      "[step = 80] loss: 0.648, auc: 0.574\n",
      "[step = 90] loss: 0.652, auc: 0.561\n",
      "\n",
      "EPOCH = 2, loss = 0.652,auc  = 0.561, val_loss = 0.581, val_auc = 0.519\n",
      "\n",
      "================================================================================2020-10-18 15:05:47\n",
      "[step = 10] loss: 0.639, auc: 0.633\n",
      "[step = 20] loss: 0.587, auc: 0.674\n",
      "[step = 30] loss: 0.564, auc: 0.680\n",
      "[step = 40] loss: 0.566, auc: 0.688\n",
      "[step = 50] loss: 0.568, auc: 0.685\n",
      "[step = 60] loss: 0.558, auc: 0.676\n",
      "[step = 70] loss: 0.568, auc: 0.661\n",
      "[step = 80] loss: 0.577, auc: 0.656\n",
      "[step = 90] loss: 0.581, auc: 0.651\n",
      "\n",
      "EPOCH = 3, loss = 0.581,auc  = 0.651, val_loss = 0.578, val_auc = 0.489\n",
      "\n",
      "================================================================================2020-10-18 15:05:49\n",
      "[step = 10] loss: 0.561, auc: 0.577\n",
      "[step = 20] loss: 0.524, auc: 0.677\n",
      "[step = 30] loss: 0.507, auc: 0.692\n",
      "[step = 40] loss: 0.533, auc: 0.704\n",
      "[step = 50] loss: 0.541, auc: 0.699\n",
      "[step = 60] loss: 0.559, auc: 0.675\n",
      "[step = 70] loss: 0.557, auc: 0.671\n",
      "[step = 80] loss: 0.558, auc: 0.676\n",
      "[step = 90] loss: 0.560, auc: 0.678\n",
      "\n",
      "EPOCH = 4, loss = 0.560,auc  = 0.678, val_loss = 0.545, val_auc = 0.430\n",
      "\n",
      "================================================================================2020-10-18 15:05:50\n",
      "[step = 10] loss: 0.537, auc: 0.747\n",
      "[step = 20] loss: 0.535, auc: 0.718\n",
      "[step = 30] loss: 0.510, auc: 0.706\n",
      "[step = 40] loss: 0.534, auc: 0.691\n",
      "[step = 50] loss: 0.528, auc: 0.705\n",
      "[step = 60] loss: 0.525, auc: 0.700\n",
      "[step = 70] loss: 0.532, auc: 0.701\n",
      "[step = 80] loss: 0.527, auc: 0.696\n",
      "[step = 90] loss: 0.526, auc: 0.705\n",
      "\n",
      "EPOCH = 5, loss = 0.526,auc  = 0.705, val_loss = 0.769, val_auc = 0.404\n",
      "\n",
      "================================================================================2020-10-18 15:05:51\n",
      "[step = 10] loss: 0.554, auc: 0.708\n",
      "[step = 20] loss: 0.540, auc: 0.727\n",
      "[step = 30] loss: 0.518, auc: 0.741\n",
      "[step = 40] loss: 0.503, auc: 0.742\n",
      "[step = 50] loss: 0.512, auc: 0.733\n",
      "[step = 60] loss: 0.502, auc: 0.728\n",
      "[step = 70] loss: 0.495, auc: 0.742\n",
      "[step = 80] loss: 0.510, auc: 0.733\n",
      "[step = 90] loss: 0.504, auc: 0.745\n",
      "\n",
      "EPOCH = 6, loss = 0.504,auc  = 0.745, val_loss = 0.593, val_auc = 0.482\n",
      "\n",
      "================================================================================2020-10-18 15:05:53\n",
      "Finished Training...\n"
     ]
    }
   ],
   "source": [
    "epochs = 6\n",
    "log_step_freq = 10\n",
    "\n",
    "dfhistory = pd.DataFrame(columns=[\"epoch\", \"loss\", metric_name, \"val_loss\", \"val_\"+metric_name])\n",
    "print('Start Training...')\n",
    "nowtime = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "print('========='*8 + \"%s\" %nowtime)\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    # 训练阶段\n",
    "    net.train()\n",
    "    loss_sum = 0.0\n",
    "    metric_sum = 0.0\n",
    "    step = 1\n",
    "    \n",
    "    for step, (features, labels) in enumerate(dl_train, 1):\n",
    "        \n",
    "        # 梯度清零\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # 正向传播\n",
    "        predictions = net(features)\n",
    "        loss = loss_func(predictions, labels)\n",
    "        try:          # 这里就是如果当前批次里面的y只有一个类别， 跳过去\n",
    "            metric = metric_func(predictions, labels)\n",
    "        except ValueError:\n",
    "            pass\n",
    "        \n",
    "        # 反向传播求梯度\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # 打印batch级别日志\n",
    "        loss_sum += loss.item()\n",
    "        metric_sum += metric.item()\n",
    "        if step % log_step_freq == 0:\n",
    "            print((\"[step = %d] loss: %.3f, \"+metric_name+\": %.3f\") %\n",
    "                  (step, loss_sum/step, metric_sum/step))\n",
    "    \n",
    "    # 验证阶段\n",
    "    net.eval()\n",
    "    val_loss_sum = 0.0\n",
    "    val_metric_sum = 0.0\n",
    "    val_step = 1\n",
    "    \n",
    "    for val_step, (features, labels) in enumerate(dl_vaild, 1):\n",
    "        with torch.no_grad():\n",
    "            predictions = net(features)\n",
    "            val_loss = loss_func(predictions, labels)\n",
    "            try:\n",
    "                val_metric = metric_func(predictions, labels)\n",
    "            except ValueError:\n",
    "                pass\n",
    "        val_loss_sum += val_loss.item()\n",
    "        val_metric_sum += val_metric.item()\n",
    "    \n",
    "    # 记录日志\n",
    "    info = (epoch, loss_sum/step, metric_sum/step, val_loss_sum/val_step, val_metric_sum/val_step)\n",
    "    dfhistory.loc[epoch-1] = info\n",
    "    \n",
    "    # 打印epoch级别日志\n",
    "    print((\"\\nEPOCH = %d, loss = %.3f,\"+ metric_name + \\\n",
    "          \"  = %.3f, val_loss = %.3f, \"+\"val_\"+ metric_name+\" = %.3f\") \n",
    "          %info)\n",
    "    nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "    print(\"\\n\"+\"==========\"*8 + \"%s\"%nowtime)\n",
    "        \n",
    "print('Finished Training...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:06:05.718597Z",
     "start_time": "2020-10-18T07:06:05.453751Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUZfbA8e9JQlUpAiKCBFBAKUqJoIvYRVQE1wqiP7GxgNjWAiI2BMu61pVVsa9gW+yra1vBsmuhiCJIEykRkRC6dDi/P86NGcIkJGQmd8r5PM88mblzZ+ZMIHPmbecVVcU555wrKiPsAJxzziUmTxDOOeei8gThnHMuKk8QzjnnovIE4ZxzLipPEM4556LyBOEqhIhkisg6EWkcy3PDJCIHikjM54mLyAkisiDi9mwR6Vqac3fjtZ4UkWG7+/gSnnekiDwb6+d1FSsr7ABcYhKRdRE3qwObgG3B7T+p6riyPJ+qbgP2jPW56UBVW8bieUTkUuB8VT0m4rkvjcVzu9TkCcJFpaq/f0AH31AvVdWPijtfRLJUdWtFxOacqxjexeR2S9CF8LKIvCgia4HzReQIEflSRFaJyC8i8rCIVArOzxIRFZEmwe2xwf3/FpG1IvKFiDQt67nB/SeLyBwRWS0ifxOR/4pIv2LiLk2MfxKReSKyUkQejnhspog8ICL5IvIj0L2E389wEXmpyLHRInJ/cP1SEfkheD8/Bt/ui3uuXBE5JrheXUSeD2KbAXSM8rrzg+edISI9g+NtgUeArkH33fKI3+1tEY8fELz3fBF5Q0QalOZ3sysicnoQzyoR+VhEWkbcN0xElojIGhGZFfFeDxeRqcHxX0Xk3tK+nosRVfWLX0q8AAuAE4ocGwlsBk7DvmhUAw4DOmMt02bAHGBwcH4WoECT4PZYYDmQA1QCXgbG7sa5+wBrgV7BfX8GtgD9inkvpYnxTaAm0ARYUfDegcHADKARUAf41P6Eor5OM2AdsEfEcy8DcoLbpwXnCHAcsAE4JLjvBGBBxHPlAscE1/8KTARqA9nAzCLnngM0CP5NzgtiqB/cdykwsUicY4HbguvdghjbAVWBvwMfl+Z3E+X9jwSeDa4fHMRxXPBvNCz4vVcCWgMLgX2Dc5sCzYLrk4A+wfW9gM5h/y2k28VbEK48PlfVt1V1u6puUNVJqvqVqm5V1fnAGODoEh4/XlUnq+oWYBz2wVTWc3sA01T1zeC+B7BkElUpY7xLVVer6gLsw7jgtc4BHlDVXFXNB+4u4XXmA99jiQvgRGCVqk4O7n9bVeer+Rj4DxB1ILqIc4CRqrpSVRdirYLI131FVX8J/k1ewJJ7TimeF6Av8KSqTlPVjcBQ4GgRaRRxTnG/m5L0Bt5S1Y+Df6O7gRpYot6KJaPWQTflT8HvDizRNxeROqq6VlW/KuX7cDHiCcKVx+LIGyJykIi8IyJLRWQNMAKoW8Ljl0ZcX0/JA9PFnbtfZByqqtg37qhKGWOpXgv75luSF4A+wfXzsMRWEEcPEflKRFaIyCrs23tJv6sCDUqKQUT6ici3QVfOKuCgUj4v2Pv7/flUdQ2wEmgYcU5Z/s2Ke97t2L9RQ1WdDVyL/TssC7os9w1OvQhoBcwWka9F5JRSvg8XI54gXHkUneL5OPat+UBVrQHcgnWhxNMvWJcPACIi7PiBVlR5YvwF2D/i9q6m4b4MnBB8A++FJQxEpBowHrgL6/6pBXxQyjiWFheDiDQDHgUGAnWC550V8by7mpK7BOu2Kni+vbCurJ9LEVdZnjcD+zf7GUBVx6pqF6x7KRP7vaCqs1W1N9aNeB/wqohULWcsrgw8QbhY2gtYDfwmIgcDf6qA1/wX0EFEThORLOAqoF6cYnwFuFpEGopIHWBISSer6q/A58AzwGxVnRvcVQWoDOQB20SkB3B8GWIYJiK1xNaJDI64b08sCeRhufJSrAVR4FegUcGgfBQvApeIyCEiUgX7oP5MVYttkZUh5p4ickzw2tdj40ZficjBInJs8Hobgss27A1cICJ1gxbH6uC9bS9nLK4MPEG4WLoWuBD7438c+wYdV8GH8LnA/UA+cADwDbZuI9YxPoqNFUzHBlDHl+IxL2CDzi9ExLwKuAZ4HRvoPQtLdKVxK9aSWQD8G/hHxPN+BzwMfB2ccxAQ2W//ITAX+FVEIruKCh7/HtbV83rw+MbYuES5qOoM7Hf+KJa8ugM9g/GIKsBfsHGjpViLZXjw0FOAH8Rmyf0VOFdVN5c3Hld6Yl22zqUGEcnEujTOUtXPwo7HuWTmLQiX9ESku4jUDLopbsZmxnwdcljOJT1PEC4VHAnMx7opugOnq2pxXUzOuVLyLibnnHNReQvCOedcVClTrK9u3brapEmTsMNwzrmkMmXKlOWqGnVqeMokiCZNmjB58uSww3DOuaQiIsVWBPAuJuecc1F5gnDOOReVJwjnnHNRpcwYhHMuNW3ZsoXc3Fw2btwYdihJrWrVqjRq1IhKlYorxbUzTxDOuYSWm5vLXnvtRZMmTbBiva6sVJX8/Hxyc3Np2rTprh8QSPsupnHjoEkTyMiwn+PG7eoRzrmKtHHjRurUqePJoRxEhDp16pS5FZbWLYhx46B/f1i/3m4vXGi3AfqWu4alcy5WPDmU3+78DtO6BXHTTYXJocD69XbcOefSXVoniEWLynbcOefSSVwTRFCGebaIzBORoVHubywiE0TkGxH5rmDPWRFpIiIbRGRacHksHvE1LmbDyOKOO+cSX6zHFVetWsXf//73Mj/ulFNOYdWqVWV+XL9+/Rg/vjR7UcVf3BJEsHHLaOBkbOPxPiLSqshpw4FXVLU90BuI/Ff4UVXbBZcB8Yhx1CioXn3HY9Wr23HnXPIpGFdcuBBUC8cVy5MkiksQ27ZtK/Fx7777LrVq1dr9F04A8WxBdALmqer8YJvAl7CN2yMpUCO4XhPbCazC9O0LY8ZAdnbhsWuv9QFq5xLZMcfsfCn4/L7xxujjilddZdeXL9/5sbsydOhQfvzxR9q1a8dhhx3Gsccey3nnnUfbtm0BOP300+nYsSOtW7dmzJgxvz+uSZMmLF++nAULFnDwwQdz2WWX0bp1a7p168aGDRtK9V7/85//0L59e9q2bcvFF1/Mpk2bfo+pVatWHHLIIVx33XUA/POf/6RNmzYceuihHHXUUaV6/l1S1bhcsH12n4y4fQHwSJFzGmD7++YCK4GOwfEmwG/Y3sKfAF2LeY3+wGRgcuPGjbU8fvtNtUED1e7dy/U0zrkYmzlz5g63jz5658vo0XafiKq1HXa+qKrm5e382F356aeftHXr1qqqOmHCBK1evbrOnz//9/vz8/NVVXX9+vXaunVrXb58uaqqZmdna15env7000+amZmp33zzjaqqnn322fr8888X+3oXXnih/vOf/9QNGzZoo0aNdPbs2aqqesEFF+gDDzyg+fn52qJFC92+fbuqqq5cuVJVVdu0aaO5ubk7HCuq6O9SVRWYrMV8jsdzmmu0OVVFdyfqAzyrqveJyBHA8yLShmDDdFXNF5GOwBsi0lpV1+zwZKpjgDEAOTk55dr5qHp1GD8eWrQoz7M45+Jt4sTi72vc2LqViiroJahbt+THl0anTp12WGz28MMP8/rrrwOwePFi5s6dS506dXZ4TNOmTWnXrh0AHTt2ZMGCBbt8ndmzZ9O0aVNaBB9KF154IaNHj2bw4MFUrVqVSy+9lFNPPZUePXoA0KVLF/r168c555zDGWecUb43GYhnF1MusH/E7Ubs3IV0CfAKgKp+AVQF6qrqJlXND45PAX4E4v7R/Yc/2H8gVdi+Pd6v5pyLtYoYV9xjjz1+vz5x4kQ++ugjvvjiC7799lvat28fdTFalSpVfr+emZnJ1q1bd/k6Wsxun1lZWXz99deceeaZvPHGG3Tv3h2Axx57jJEjR7J48WLatWtHfn5+Wd/aTuKZICYBzUWkqYhUxgah3ypyziLgeAARORhLEHkiUi8Y5EZEmgHNsT2H427pUujcGV5+uSJezTkXS5HjiiL2c8yY8o0r7rXXXqxduzbqfatXr6Z27dpUr16dWbNm8eWXX+7+CxVx0EEHsWDBAubNmwfA888/z9FHH826detYvXo1p5xyCg8++CDTpk0D4Mcff6Rz586MGDGCunXrsnjx4nLHELcuJlXdKiKDgfeBTOBpVZ0hIiOwPq+3gGuBJ0TkGqz7qZ+qqogcBYwQka3ANmCAqq6IV6yR9tkHNm2Cm2+Gs86CMtS1cs4lgL59YzvRpE6dOnTp0oU2bdpQrVo16tev//t93bt357HHHuOQQw6hZcuWHH744TF73apVq/LMM89w9tlns3XrVg477DAGDBjAihUr6NWrFxs3bkRVeeCBBwC4/vrrmTt3LqrK8ccfz6GHHlruGKS4ZkyyycnJ0VjtKPfOO9CjBzz6KAyIywRb51xp/fDDDxx88MFhh5ESov0uRWSKquZEOz+tV1IX55RToEsXGDFi5ylzzjmXLjxBRCECd98Nv/wCTz4ZdjTOuVR0+eWX065dux0uzzzzTNhh7SCtq7mW5Mgj4bXXrDXhnHOxNnr06LBD2CVPECX44x/tp6q1KpxzLp14F9MuTJgArVrZ9FfnnEsnniB2Yf/9Yd48L+DnnEs/niB24cAD4ZJL4PHH4aefwo7GOecqjieIUrj5ZsjMhFtvDTsS59wuhbzR/J577lnsfQsWLKBNmzYVGE35eIIohYYN4YorYOxYmDMn7Gicc8WKx4YQacxnMZXS0KG2eK5587AjcS6NXX01BLWHovryS6uVE2n9eusnfuKJ6I9p1w4efLDYpxwyZAjZ2dkMGjQIgNtuuw0R4dNPP2XlypVs2bKFkSNH0qtX0e1uSrZx40YGDhzI5MmTycrK4v777+fYY49lxowZXHTRRWzevJnt27fz6quvst9++3HOOeeQm5vLtm3buPnmmzn33HPL9Hq7wxNEKe29NxT8+/u0V+cSVNHksKvjpdC7d2+uvvrq3xPEK6+8wnvvvcc111xDjRo1WL58OYcffjg9e/ZEyvDBULAOYvr06cyaNYtu3boxZ84cHnvsMa666ir69u3L5s2b2bZtG++++y777bcf77zzDmBFAiuCJ4gyeuABm/r65pueJJyrcCV80wdszKG4DSF2cyOI9u3bs2zZMpYsWUJeXh61a9emQYMGXHPNNXz66adkZGTw888/8+uvv7LvvvuW+nk///xzrrjiCsAqt2ZnZzNnzhyOOOIIRo0aRW5uLmeccQbNmzenbdu2XHfddQwZMoQePXrQtWvX3XovZeVjEGWUlQVvvw0ffhh2JM65ncRpQ4izzjqL8ePH8/LLL9O7d2/GjRtHXl4eU6ZMYdq0adSvXz/qPhAlKa5Q6nnnncdbb71FtWrVOOmkk/j4449p0aIFU6ZMoW3bttx4442MGDGiXO+ntDxBlFH//vZlZNgw62pyziWQeGwIgXUzvfTSS4wfP56zzjqL1atXs88++1CpUiUmTJjAwmitll046qijGBcMns+ZM4dFixbRsmVL5s+fT7Nmzbjyyivp2bMn3333HUuWLKF69eqcf/75XHfddUydOrVc76e0vIupjKpUgdtvh3794NVXbc8I51wCifWGEEDr1q1Zu3YtDRs2pEGDBvTt25fTTjuNnJwc2rVrx0EHHVTm5xw0aBADBgygbdu2ZGVl8eyzz1KlShVefvllxo4dS6VKldh333255ZZbmDRpEtdffz0ZGRlUqlSJRx99NKbvrzi+H8Ru2LYNDjnEtiWdMcOmWzvn4sP3g4idsu4H4S2I3ZCZaa3W6tU9OTjnUpcniN3UpUvYETjnEtX06dO54IILdjhWpUoVvvrqq5Ai2j2eIMph0ya47DLIyYErrww7GudSl6qWaY1B2Nq2bcu0khb0hWB3hhPi2kEiIt1FZLaIzBORoVHubywiE0TkGxH5TkROibjvxuBxs0XkpHjGubuqVLEy4HfcAWvWhB2Nc6mpatWq5Ofn79YHnDOqSn5+PlWrVi3T4+LWghCRTGA0cCKQC0wSkbdUdWbEacOBV1T1URFpBbwLNAmu9wZaA/sBH4lIC1XdFq94d9edd8Jhh8H998Ntt4UdjXOpp1GjRuTm5pKXlxd2KEmtatWqNGrUqEyPiWcXUydgnqrOBxCRl4BeQGSCUKBGcL0msCS43gt4SVU3AT+JyLzg+b6IY7y7JSfHprredx9cfjnUqxd2RM6llkqVKtG0adOww0hL8exiaggsjridGxyLdBtwvojkYq2HK8rwWESkv4hMFpHJYX67uOMOqwd2zz2hheCcczEXzwQRbUSpaCdiH+BZVW0EnAI8LyIZpXwsqjpGVXNUNadeiF/dDzoInn0WrrkmtBCccy7m4tnFlAvsH3G7EYVdSAUuAboDqOoXIlIVqFvKxyaUIjPanHMu6cWzBTEJaC4iTUWkMjbo/FaRcxYBxwOIyMFAVSAvOK+3iFQRkaZAc+DrOMYaE3PnwvHHw6xZYUfinHPlF7cEoapbgcHA+8AP2GylGSIyQkR6BqddC1wmIt8CLwL91MwAXsEGtN8DLk/EGUxF1aoFX39tW5Q651yy81pMMXbbbVbMb9Ikm+HknHOJrKRaTF5JKMb+/GeoU8fKgTvnXDLzBBFjNWpYcvjwQ9t5zjnnkpXXYoqDQYNsr5JOncKOxDnndp8niDioWtXXRDjnkp93McXRu+9Cr162wZBzziUbTxBx9Ntv8NZbEGw765xzScUTRBydeSZ06AC33gqbN4cdjXPOlY0niDjKyIC77oIFC2yLUuecSyaeIOLsxBPhmGOs4uu6dWFH45xzpecJIs5E4N574cEHoXr1sKNxLoWNGwdNmljTvUkTH/yLAZ/mWgFycrzshnNxNW4c9O9vG7MALFxotwH69g0vriTnLYgKdO+9MGpU2FE4l4JuuqkwORRYv96Ou93mCaICTZ8OI0fCzz+HHYlzKWbRorIdd6XiCaIC3X67LZq7446wI3EuxTRuXLbjrlQ8QVSgpk2tW/Spp2DevLCjcS6FXH31zsdE7FuZ222eICrY8OFQuTLcckvYkTiXQn74ATIzoWFDSwz16oGq9+eWkyeICrbvvvDII3D55WFH4lyKyM2FZ56x5nluLmzfDsuWwVlnWX/u/PlhR5i0PEGE4KKLoEuXsKNwLkXce6+1Fm64YcfjDz4IWVkweLDd78rME0RIVq6EgQPh88/DjsS5JLZ0qdWxueACWxwXqWFDmzb473/Da6+FEl6yi2uCEJHuIjJbROaJyNAo9z8gItOCyxwRWRVx37aI+96KZ5xhqFIF3nwThg71LzfO7bb777dKmDfeGP3+yy+Hdu3gyith7dqKjS0FxC1BiEgmMBo4GWgF9BGRVpHnqOo1qtpOVdsBfwMi0/yGgvtUtWe84gxL9eo2UP3f/9q+Ec65MsrPh7//HXr3hubNo5+TlQWPPQa//OIzQ3ZDPFsQnYB5qjpfVTcDLwG9Sji/D/BiHONJOJdcAgccYIs9t28POxrnksxDD9mmK8OGlXxe584wYAA8/DB8803FxJYi4pkgGgKLI27nBsd2IiLZQFPg44jDVUVksoh8KSKnF/O4/sE5k/Py8mIVd4WpVAlGjIBvv4WXXw47GueSyOrV9oF/xhnQuvWuz7/zTqhbF/70J9/isQzimSAkyrHiett7A+NVNfJfrrGq5gDnAQ+KyAE7PZnqGFXNUdWcevXqlT/iEPTubd2nhx8ediTOJZFHHrEkMXx46c6vVQseeAAmTfLNWcogngkiF9g/4nYjYEkx5/amSPeSqi4Jfs4HJgLtYx9i+DIy7MtN06ZhR+Jckli3zj7sTz0V2pfhY6FPHzj+ePtG9uuv8YsvhcQzQUwCmotIUxGpjCWBnWYjiUhLoDbwRcSx2iJSJbheF+gCzIxjrKGbOdNm6hUtSOmcK+Kxx2yAurSthwIiNqi9YQNce218YksxcUsQqroVGAy8D/wAvKKqM0RkhIhEzkrqA7ykusNkz4OBySLyLTABuFtVUzpB5OfD2LHWcnbOFWPDBvjrX+GEE3avX7ZFC5tbPm4c/Oc/sY8vxYimyCT8nJwcnTx5cthhlMspp8CXX1plgFq1wo7GuQT0yCNwxRUwcSIcffTuPcfGjdCmjfXvfvcdVK0a0xCTjYhMCcZ7d+IrqRPInXfaCuu//jXsSJxLQJs3wz33wJFHwlFH7f7zVK1qXU1z58Jf/hK7+FKQJ4gE0q6dzWp64AGrIOCci/Dcc1aMb/hwG08oj27d7I/tzjstUbioPEEkmDvugD//2VZaO+cCW7fCXXfBYYfZh3ss3H+/1by5/HKvd1MMTxAJ5sADLUnUqBF2JM4lkBdfhJ9+ik3roUCDBrZJ/IcfwiuvxOY5U4wPUieod96ByZPh1lvDjsS5kG3bZqulq1SBadNilyAKnrtzZ9tYaNYsqFkzds+dJHyQOgl98ontlvj992FH4lzIXn0VZs+ObeuhQGYmPP64bTBU1nUVacATRIIaMgT22sv/z7o0t3277elw0EFWdykeOna0cYjRo63Z7n7nCSJB1aljG2S9+aatjXAuLf3rXzB9ulVszcyM3+vccYftB+zF/HbgCSKBXXUV7LOPlY5JkaEi50pP1T64mzWzOkrxVLOmbVE6daqtkXCAJ4iEtueetmju/PM9Qbg09MEH1uVz44228U+8nX02nHSSbdCypLi6ounFZzE55xKPKnTtCgsXwo8/QuXKFfO68+ZZGY7TT4eXXqqY1wyZz2JKclu22N4o//pX2JE4V0E++cT24x0ypOKSA9hCpJtush283n+/4l43QXkLIgls2waHHGI/v/++YlrbzoXqhBNgxgyrXFmtWsW+9qZNhX9w06dX/OtXMG9BJLnMTJvpN3u2laNxLqV98YWV4r7uunA+nKtUgUcfta6tu+6q+NdPIN6CSBKqVv5+yRKrLZbmFYpdKuvRw+Z2L1hgMzXCcsEF1tU0fTq0bBleHHHmLYgUIGJfZnJz7cuNcylp6lSrM3PNNeEmB7AphHvsAYMGpe00Qk8QSeS44+Dqq8u2Da9zSWXUKFuTMHhw2JFA/fr2rezjj+GFF8KOJhTexeScSwwzZtgU05tvhhEjwo7GbN8Of/iDVZKdNQtq1w47opjzLqYUk59vs//y8sKOxLkYGjXKupWuuirsSAplZMBjj8Hy5VbuI83ENUGISHcRmS0i80RkaJT7HxCRacFljoisirjvQhGZG1wujGecyWbZMuseTfMJFi6VzJljA8KDBlkhskTSrp0lrccfT7vCaHHrYhKRTGAOcCKQC0wC+qjqzGLOvwJor6oXi8jewGQgB1BgCtBRVVcW93rp1sV08cUwbpzNaGrcOOxonCuniy+2TYEWLLC+/0Szdi0cfDDUrWvlP1JoMVJYXUydgHmqOl9VNwMvAb1KOL8P8GJw/STgQ1VdESSFD4HucYw16dx2m/28/fZQw3Cu/BYsgOefh/79EzM5gNXef/hh+PZb+Nvfwo6mwsQzQTQEFkfczg2O7UREsoGmwMdleayI9BeRySIyOS/NOuQbN7bW+LPP2tiZc0nrnnusr//668OOpGR//COccgrccovNN08D8UwQ0bZ+Kq4/qzcwXlULCrGX6rGqOkZVc1Q1p169ersZZvIaNgx6967YUjXOxdTPP8PTT8NFF0GjRmFHUzIReOQR2LrV5pungXgmiFxg/4jbjYDiauj2prB7qayPTVv16tk4RLNmYUfi3G7661+t5tGQIWFHUjpNm1oL4tVXbUFfiotngpgENBeRpiJSGUsCbxU9SURaArWBLyIOvw90E5HaIlIb6BYcc1HMmmUzBJ1LKsuW2cyg88+3D95kce21NmA9eDCsXx92NHEVtwShqluBwdgH+w/AK6o6Q0RGiEjPiFP7AC9pxHQqVV0B3IElmUnAiOCYi+Ldd23v6gkTwo7EuTK4/37YuDH51hdUrmxrIxYssCqaKaxU01xF5CrgGWAt8CTQHhiqqh/EN7zSS7dprpE2boTmzaFhQyuEKdFGcJxLJCtWQHa2FeZ78cVdn5+ILroIxo61mU2tWoUdzW6LxTTXi1V1DdbVUw+4CLg7RvG5cqpaFW69Fb76Ct7aqRPPuQT08MOwbl3ytR4i/eUvUKMGDByYssX8SpsgCr6TngI8o6rfEn2mkQtJv37QooVthrVt2y5Pdy48a9bAQw/ZtNG2bcOOZvfVq2dTdD/9FP7xj7CjiYvSJogpIvIBliDeF5G9gO3xC8uVVVYW3H237bm+aVPY0ThXgtGjYdUq+zaT7C6+2Ir5XXedFUlLMaUdg8gA2gHzVXVVUAqjkap+F+8ASyudxyCcSxq//QZNmsBhh9nsilQwfbrV4L/oInjiibCjKbNYjEEcAcwOksP5wHBgdawCdLGjCh99ZNO0nUs4jz9ulVGHDw87kthp2xb+/Gd48kn473/DjiamStuC+A44FDgEeB54CjhDVY+Ob3il5y0IowrHHgs//GBb6oa9KZdzv9u40dY7tGple06nkt9+s/dVo4btilepUtgRlVosWhBbg3UKvYCHVPUhYK9YBehip2Br0mXLbBzQuYTx9NOwdGlqtR4K7LGHFfH7/nt48MGwo4mZ0rYgPgHeAy4GugJ5wDRVTZgpCN6C2FGvXjBxIsyfn3jl9V0a2rzZFus0agSff566i3V69bI+3h9+SJo6/LFoQZwLbMLWQyzFKqveG6P4XByMGmUl7O+5J+xInMPKeS9aZNuJpmpyAFvfAXDlleHGESOlShBBUhgH1BSRHsBGVU3Nib8pok0bKzh5wAFhR+LS3tat1u/ZsaPNw05l2dm2Wcubb9olyZW2i+kcrMUwEVsg1xW4XlXHxzW6MvAuJucS1LhxVpDv9dfh9NPDjib+tmyBDh1g9WqYOTPhZ4rEoovpJuAwVb1QVf8P2y3u5lgF6OJn61Z46imYNy/sSFxa2r7d+jvbtIGePXd9fiqoVMmK+S1eDCNGhB1NuZQ2QWSo6rKI2/lleKwL0fLl1h16yy1hR+LS0muv2YDt8OG2a1y66NIFLr3UKtZOnx52NLuttP9i74nI+xQ309IAAB9zSURBVCLST0T6Ae8AKbIMMrXtuy9cdZUVzJw2LexoXFpRtXLYLVrAWWeFHU3Fu/tuqF3bivltT87KRKUdpL4eGIMtlDsUGKOqSbIFlLvhBvt/mgqlb1wSeecdK4U9bBhkZoYdTcWrUwfuvddWVz/zTNjR7JZSDVInAx+kLtk998DQofDZZ3DkkWFH41KeKhx+OOTlwezZSbWyOKZU4ZhjbAHdrFlWATbB7PYgtYisFZE1US5rRWRNfMKtYOPGWfGwjAz7OW5c2BHFxRVXwIknJm1L1yWbjz6Cr7+2byXpmhzA1nw8+qiVOL/hhrCjKbP0bkGMGwf9+++4r2z16jBmDPTtG9sAnUsnRx9txcB+/BGqVAk7mvANG2ZrQSZOtN9NAonFNNfUdNNNO286vn59SnfWr14NF1xg63lSvNHkwvLpp3a54QZPDgWGD7c/toEDrexIkohrghCR7iIyW0TmicjQYs45R0RmisgMEXkh4vg2EZkWXOKzkeaiRWU7ngJuusm20V20yLpHFy60RpQnCRczI0fCPvvAZZeFHUniqF7dNkr64Qe4776woym1uCUIEckERgMnA62APiLSqsg5zYEbgS6q2hq4OuLuDaraLrjEZ4VNccW0atdO2T1m335752Mp3mhyFemrr+DDD22HtWrVwo4msZxyCpxxBtxxB/z0U9jRlEo8WxCdgHmqOl9VNwMvYeXCI10GjFbVlQBFFuPF36hRltkjZWTAihVw8smwZEmFhlMRFi+OfjyFG02uIo0aBXvvDQMGhB1JYnroIZvyO3hwUnwJjWeCaAhEfhzlBscitQBaiMh/ReRLEekecV9VEZkcHI9awEVE+gfnTM7Lyyt7hH372oB0drbNNsjOts3HR4+2PtQ2beDll8v+vAmsuEZTklQmdols2jRrol5zDezl28VE1aiRld94912rTZXoVDUuF+Bs4MmI2xcAfytyzr+A14FKQFMsidQK7tsv+NkMWAAcUNLrdezYUWNq9mzVzp1VQbV3b9X8/Ng+f0jGjlWtXt3eVsGlenXV0aNVv/wy7OhcUjvrLNUaNVRXrgw7ksS2ZYvqoYeqNmyoumZN2NEoMFmL+VyNZwsiF9g/4nYjoGifTS7wpqpuUdWfgNlAcwBVXRL8nI9VkW0fx1h31qKFbWwyciSMH2/7zr7/foWGEA/RGk1jxli5mCOOsK11f/st7Chd0pk50zZCv+IKqFUr7GgSW1aW7c29ZAncemvY0ZSsuMxR3guQBczHWgaVgW+B1kXO6Q48F1yvi3VJ1QFqA1Uijs8FWpX0ejFvQUSaMkW1VSv7uj1okOq6dfF7rZCsXq06cKC9xaZNVT/6KOyIXFLp21d1jz1U8/LCjiR5DBigmpGh+s03oYZBGC0IVd0KDAbeB34AXlHVGSIyQkQKZiW9D+SLyExgArbHRD5wMDBZRL4Njt+tqjPjFesudegAU6bY1+tHH4X27eHLL0MLJx5q1IC//x0++cS+4JxwAjz5ZNhRuaQwb55Vgxw4EOrWDTua5HHnnfb7GjAgcUscFJc5ku0S1xZEpAkTVBs3tsw/fLjqpk0V87oVaP161ZtvVv31V7u9dm248bgEd8klqlWqqP7yS9iRJJ/nn7dm+2OPhRYCJbQg0rvUxu5avdr283z2WWtNPP88tG5dMa9dwbZts9L22dm23W79+mFH5BLKokW2r+2AAfC3v4UdTfJRheOPh2++sWJ+IfyBeamNWKtZ08r3vvaaLSzo2NE2BknUZmI5qNpGYG+8Aa1aWS5Mke8ULhbuucdmOyRhIbqEUFDMb/16W1yYYDxBlMcf/2hlfE86Ca69Fo47zmpXpJCsLKszNm0atGwJ//d/cOqpVsXZpbklS2w/2379YP/9d3m6K0bLljBkiNXA+fjjsKPZgSeI8qpf375eP/00TJ1q02GffTblvmYffLDtJfHww7B0acLvw+4qwn332abnQ6OWWXNlceON1lU3cCBs2hR2NL/zBBELInDRRfDddzYmcdFF1rpYVrGVQ+ItM9OmuU+ebGV2fvvN1lXMnh12ZK7C5eXBY4/BeedBs2ZhR5P8qlWzCg5z5sBf/hJ2NL/zBBFLTZrAhAnw17/Cv/9tpTrefDPsqGKuYO/577+3igGHHmrb727ZEm5crgI98ABs2GD9jy42TjoJzjnH6lnNmxd2NIAniNjLyLDxiClToGFDOP10uPhi21EqxXTubNWLe/SwFnLnzjZW4VLcypXwyCNw9tlw0EFhR5NaHngAKleGyy9PiG5qTxDx0qaNlT4eNgyeew4OOcRWoaWYffe1SiTjx9uY5Z//HHZELu4efhjWrvUa8fGw337WgvjgA/jnP8OOxtdBVIgvvrDpPz/+aJ+gI0dC1aphRxVzK1bAunVWGXbpUnu7XbqEHZWLqTVrrCv1qKNscoaLvW3brDm+ZIk10WvWjOvL+TqIsB1xhPW9DBhgMz86drSFMSlm770Ly4bfcQd07WqD2mvXhhuXi6FHH7UuJm89xE9mpk0AWLoUbr451FA8QVSUPfawYkf//rf9gXXqZE3JrVvDjiwu7rnHksPo0dbblgKFcN369fYF56ST4LDDwo4mteXk2DjE6NE2nhkSTxAVrXt3q619xhm2kXnXrjB3bthRxdyee9rmWZ99Zpv2de9uZcVdEhszxqa3Dh8ediTpoWBv7wEDrNspBJ4gwlCnju1U98ILVn+lXTtruqfIeFCkLl2sN+32221pCHiXU1LauBHuvReOOQaOPDLsaNJDzZo2q2nyZOtyCoEniDD16WOLCY48EgYNsn2wf/457KhirmpVuOUWqFfPvgideCKceSb88kvYkblSe/ZZGzT11kPFOvdc+4MZNiyUPxhPEGFr2BDee69wH+y2beGll8KOKm5UrXftnXes+N8zz6Rkwym1bNliKyEPP9zqjbmKI2KfDZs2hTKH3BNEIhCxFsS0abbVaZ8+dlmxIuzIYi4rywp/fved5cKLL4Zu3eDXX8OOzBVr7FgrQnnzzfZ/1VWs5s2tBfHSS7Y+ogL5OohEs3WrfVu7/XYboHr6aZs1koK2b7eteZ9+2hpP1aqFHZHbybZttlp6r71sNo0niHBs2mSLbbdts0kuMfxj8XUQySQry/p5v/zSBqm6d7fpbr/9FnZkMZeRYcUrv/qqsPhf794wM7zNZV1Rr7xidYGGD/fkEKYqVWya/I8/2hfICuIJIlF17Gjf2K65xv5jpOA+2AUKiv/NnAkffWRvdeRI2Lw53LjS3vbttlandWurKebCdfzxVj337rut6msFiGuCEJHuIjJbROaJSNSi8SJyjojMFJEZIvJCxPELRWRucLkwnnEmrGrVbKe6jz+2JmaXLvZNLkU/OQ87zJLEH/9o3d2HHWYz/FxI3ngDZsywVdMZ/l0yIdx3n30uDBpUMbM7itusurwXIBP4EWgGVAa+BVoVOac58A1QO7i9T/Bzb2B+8LN2cL12Sa/XsWPHeOznnThWrVK98ELb4Lx9e9Xvvw87orh64w3V/fZTPfbYsCNJU9u32/+z5s1Vt24NOxoX6e9/t8+BceNi8nTAZC3mczWeXws6AfNUdb6qbgZeAnoVOecyYLSqrgRQ1YIddk4CPlTVFcF9HwLd4xhr4qtZ0+aip8E+2AC9etmX1+ees9u//JKSxXAT17vv2grHG2+02kAucfTvb6V6rrnGyvbEUTwTRENgccTt3OBYpBZACxH5r4h8KSLdy/BYRKS/iEwWkcl56bJJcsE+2N26pew+2AVq1Src6njUKFvEO3BgSm6tkVhUrdpidjacf37Y0biiCor5LV8e96KJ8UwQ0aY8FO00y8K6mY4B+gBPikitUj4WVR2jqjmqmlOvXr1yhptE6te3neqeesoGslN0H+xIf/mLrRMaM8bGTN95J+yIUtjHH9vUsqFDoVKlsKNx0bRvb9UwH30UGjSwMaImTWDcuJi+TDwTRC6wf8TtRsCSKOe8qapbVPUnYDaWMErz2PQmYqvMUnwf7ALVq9v43P/+Z71tPXrY34aLg5EjbeOafv3CjsSVpE0b+xxYutS+HC5caN1PMUwS8UwQk4DmItJURCoDvYG3ipzzBnAsgIjUxbqc5gPvA91EpLaI1Aa6BcdcUU2b2je+e+9N6X2wC3TuDFOnwp13wlln2bHVq1O68VSxPv8cJk6E669PyU2tUsrIkTv/x1+/PqbdTnFLEKq6FRiMfbD/ALyiqjNEZISI9AxOex/IF5GZwATgelXNV9UVwB1YkpkEjAiOuWgyM+G666y7ab/9UnofbLAte2+8sbD4X7duNqidgnUOK97IkfaL7d8/7EjcrixaVLbju6O46U3Jdkn5aa6ltWmT6rBhqhkZqtnZqhMnhh1RXG3dqnrffarVqqnWqKE6ZozN0HS74euvbfrk3XeHHYkrjexs+/cqesnOLtPTENI0VxeGypVtys9nn1nZjmOPtdbFxo1hRxYXmZk2eD19us387d/fFpwuXRp2ZElo5EioXdsWYbnEN2qUDc5Fql7djseIJ4hU9Yc/WHXYP/0ppffBLnDAAfCf/8ATT1hNpzjv8556vv0W3noLrr7aCvO5xNe3r03ry862wersbLvdt2/MXsITRCrbc0+b6vPuuzvug/388zYlLk5T48IiApdeaiWrqlWDdetsIPvuu1Py7cbWnXdaYrjiirAjcWXRty8sWGALZhcsiGlyAFuH4FLdySdbH8ygQVbLKSOjcAV2wdQ4iPl/rrAUFB2dPdvK57/6auF9Kfh2y2/WLPjnP23dQ+3aYUfjEojvB5FOVG2PieXLd76vcmXIybGpjVWqFP+zpPvKcm4FlW/Yf3/omjuOO7mJxixiEY0Zxij+l92Xn37yCtYA/N//WRZdsMBmMLm0UtJ+EN6CSCcikJ8f/b7Nm61fZtMmmx67aZMNbEf7GQtZWWVPPLuRnC7J/YQh3EM1bJC+CQt5gv70XwhduvRl+3bo0MGGaDp0sFXalSvH5i0mhR9/hBdegKuu8uTgduItiHTTpEn02k3Z2fYNcldUbY/i4pJH0Z+lOWd3zt24sVyFCvMy6nP/n3P5YlIWU6fC2rV2/LzzbIxC1fbLbtvWLim7Zqx/f/jHP2D+fFtD49KOtyBcoVGj7ENh/frCY2WZGidiX7ET4Wv21q27TCJ67HHIzmW8qLf9V+56ah84+WS29z+Nn1p2Z/K8Wuy7r93/yy9wySV2PSvLWhYdOlhFk65dK/A9xtOiRVbD67LLPDm4qDxBpJuCkdmbbrIPiMaNLTkk44htVpZd9tij2FMku3H0FlPdunDqqfDOO2S88AIHZGVxQNeucNpp0PA0GhxwIPPnW1mPqVNtkfrbb9uykq5dbQbx+edb0ijoomrXLslmiN57rzWVhgwJOxKXoLyLyaW2ceOit5gK5otv22aVS99+2y4zZtg5Bx9syeK00+CIIyAzE1Xr1crMtKRx6632c0lQRlIEPv0UjjzSuvYXLrQ6igk5MWjpUutuPP98ePLJsKNxISqpi8kThEt948aVvsU0fz7861+WLD75xMZb6tSBU06xZHHSSVCjxg4PWbq0sKVx+eWWEO64A265xe5v1qywpXHVVTsvfg3F9dfbhlOzZ8OBB4YdjQuRJwjndsfq1baQ4q23bLHhihW2P8LRRxe2Lpo2jfrQFStsP+3ILqqlS+0ps7Lg9tutmypyBlXB+EfcLV9urYfTT4exYyvoRV2i8gThXHlt3QpffFHYFTVrlh1v3bowWXTuXOL6jnXrbHE7wG23wYsvwpw5hfd36mS9XQCTJtk+MA0bxmGtxvDhtnL6+++hVasYP7lLNp4gnIu1efMKk8Vnn1kCKRj4Pu00q0FeihHrNWusDNKUKXb76qvtZ9OmhevWCloYJ5xgg+TlsmqVTWnu1s1WT7u05wnCuXhatQree8+Sxb//bXWvKle2TbQLWhfZ2WV6yi++KOyamjrVxs4vvdRKa23bZk/Zpk1hF9UBB1gFlQLFDrsUDI58841Nu3JpzxOEcxVl61b4738LWxcFfUht2xYmi06ddvw0L4WNG61KbZ068Ouv1lCZPt0WwIM1Vh56yNZpPPOMld2KrPBevTo8/dBazh3SBLp0sXEV5/AE4Vx45swpTBaff25f//fZp7Ar6sQTCwcmymjzZpg5s7Clcd559tlfv370rclH1bqXYatusHK3nTuX8425VOEJwrlEsGLFjl1Rq1dbzahjjy1sXey/f7lfJiNj562Kq7KBBTSh/omHsu61D6hevcyNGJeiSkoQ/l/EuYqy9972Nf/FFyEvDz7+2PqC5s61BRSNG9u4wC232DSm3aw11bjxzscu4wnqswyGD+fGGy0PDRpks3gLuqmcKyquCUJEuovIbBGZJyJDo9zfT0TyRGRacLk04r5tEce9w9SllkqVrOVw//2WIGbOhHvuscGEUaNsnKJhQ6uT9NZbO64E34WiO1FWZhND5C/8etBRcNRRnHACHH44PPecrfurVw8GD47De3RJL25dTCKSCcwBTgRygUlAH1WdGXFOPyBHVXf67yki61S11J2z3sXkUkZ+vi3Me/tt65Jau9bKyR5/vHVD9ehhyaMEkbOYhtQew10r/mTNhRNP/P2cDRtsm9Y33rDV3wWlmfr1s+oiPXt6Db90EMoYhIgcAdymqicFt28EUNW7Is7phycI54q3ebMVeCoY6P7pJzveoUPhuEWHDsWvptuyBVq0sIHxL7/c5aq7ZctsoHvePLvdqZMtuD7vvDLP1HVJIqwxiIbA4ojbucGxos4Uke9EZLyIRI7QVRWRySLypYicHu0FRKR/cM7kvLy8GIbuXIKoXNlWyD30kFUA/P57uOsua1GMGGG7ADZqBH/6k9WQ2rDBHjdunJXTqFzZVtx17VqqJdn77GMTr2bMKKwAP2yYlQ0B+Pln+N//bDKWS33xbEGcDZykqpcGty8AOqnqFRHn1AHWqeomERkAnKOqxwX37aeqS0SkGfAxcLyq/ljc63kLwqWdvLzCrqj337daHtWqwUEH2Sd85OhzZAXbMvr5Zxtfr1bNKnTcdJMlkp49rXVx/PEpvKFSGkjYLqYi52cCK1S1ZpT7ngX+parji3s9TxAurW3aZNVn3367cLl1UaXdNbAEq1fbDN0334R33rHhkb33tpLnVarYy1bQduMuRsJKEFnYIPXxwM/YIPV5qjoj4pwGqvpLcP2PwBBVPVxEagPrg5ZFXeALoFfkAHdRniCcC0RbCAHWxVSObVqL2rQJJk60Lqkrgn6Bo4+2BHH66dCrl49bJINQxiBUdSswGHgf+AF4RVVniMgIEekZnHaliMwQkW+BK4F+wfGDgcnB8QnA3SUlB+dchGgLIUo6vpuqVLFpsgXJQRWOOspKgVx1lQ2BtG9vwyEuOflKaudSza520asAc+daN9Qbb9hLDhxoQyajRlnr4sgjbV8MFz4vteFcuinLLnpxpmq9Wx98YAPbmzbZuEWPHpYsune3AXAXDk8QzrmEsG6dTbh64w0b5F650pZ2NGliay9q1rSV3a7ieC0m51xC2HNPOPNMeP55G6v43/8sOQDccINtu3rUUXDffYWL9Vx4PEE450JRqZKV9Chw221w8822y95110Hz5pZMXHh8mMg5lxAOOcQut91myzXefBNq1LD7Nm2yGVHHHWfTZ48+2haJu/jyFoRzLuE0aWJTZS+6yG7n59sC8Weese2099nH6kN9+23hYwqqi2Rk2E+fXlt+niCccwlvv/3gtddg+XKrfn7mmfDRR4Wlp0aOhIsvhoULbdbUwoU209eTRPn4LCbnXFLats2mz2Zk2OynNWt2PqdxY0sWrnglzWLyMQjnXFKKrPm0dm30cxYH9aRvvtnqRXXoAB072lhH5KZKLjrvYnLOJb1dVRdZtsy6pgYPtplTNWrAWWcVnvf997ZGw+3IWxDOuaQ3alT06iIFe1o8/jg89hjk5sKUKTB1qq3mBhuz6NrVKtW2bGktjA4drIz5oYdW/HtJJJ4gnHNJr6CKSEnVRURg//3tcnrEFmTbt9vCvSlT7DJxog1uDx1qCeK33+DSSy1pFFxq167QtxcaH6R2zrkifv3VWhb77guzZ9vU2kWLCu9v1gwefhhOPdVaLevXQ9264cVbHj5I7ZxzZVC/fuH1li1tJtTy5dY1NXWqtTQKznnvPZt227hxYfdUx45WMmSPPcKJP1a8BeGcc+Uwbx68/nph4pg7147PnQsHHmg78H39dWHiaNCgVNuDVxhvQTjnXJwceCBcf33h7TVr4JtvrBsK4PPP4a67Cjf5q18fcnKslEhmps2e2mOPxEoaBbwF4ZxzcbZunZUFKZhBtWKFTbsFG8f4+usdu6dycipuu1ZvQTjnXIj23BO6dLFLUb1722D4lClw772wdauNX3zyid3/0EPWLdWxo7VKIlsa8d4XylsQzjmXIDZuhOnTYcsW+MMfLFnUrl24iK9mTWtlXHyxJYpY7Cwb2oZBItJdRGaLyDwRGRrl/n4ikici04LLpRH3XSgic4PLhfGM0znnEkHVqnDYYZYcwPbtzs+3bqknnrAKtr/9ZjOqbrppx+QAdvumm2IXT9xaECKSCcwBTgRygUlAH1WdGXFOPyBHVQcXeezewGQgB1BgCtBRVVcW93regnDOpZOMjMKB70gitvivtMJqQXQC5qnqfFXdDLwE9CrlY08CPlTVFUFS+BDoHqc4nXMu6eyq/lQsxDNBNAQWR9zODY4VdaaIfCci40Vk/7I8VkT6i8hkEZmcl5cXq7idcy7hjRq1c0XayPpTsRDPBBFtVm/RBtHbQBNVPQT4CHiuDI9FVceoao6q5tSrV69cwTrnXDLp29cGpLOzrVspO7vsA9S7Es9prrnA/hG3GwFLIk9Q1fyIm08A90Q89pgij50Y8widcy6J9e0b24RQVDxbEJOA5iLSVEQqA72BtyJPEJEGETd7Aj8E198HuolIbRGpDXQLjjnnnKsgcWtBqOpWERmMfbBnAk+r6gwRGQFMVtW3gCtFpCewFVgB9Aseu0JE7sCSDMAIVV0Rr1idc87tzBfKOedcGgttoZxzzrnk5QnCOedcVCnTxSQiecDCcjxFXWB5jMJJFun2ntPt/YK/53RRnvecrapR1wmkTIIoLxGZXFw/XKpKt/ecbu8X/D2ni3i9Z+9ics45F5UnCOecc1F5gig0JuwAQpBu7znd3i/4e04XcXnPPgbhnHMuKm9BOOeci8oThHPOuajSPkGIyNMiskxEvg87loogIvuLyAQR+UFEZojIVWHHFG8iUlVEvhaRb4P3fHvYMVUUEckUkW9E5F9hx1IRRGSBiEwPtjBOi9o7IlIr2E9nVvB3fUTMnjvdxyBE5ChgHfAPVW0TdjzxFlTQbaCqU0VkL2w719Mjt4JNNSIiwB6quk5EKgGfA1ep6pchhxZ3IvJnbOveGqraI+x44k1EFmDbGKfNQjkReQ74TFWfDCpnV1fVVbF47rRvQajqp1gl2bSgqr+o6tTg+lqsxHq0nf5Shpp1wc1KwSXlvxmJSCPgVODJsGNx8SEiNYCjgKcAVHVzrJIDeIJIayLSBGgPfBVuJPEXdLVMA5Zh+52n/HsGHgRuAMqwhX3SU+ADEZkiIv3DDqYCNAPygGeCrsQnRWSPWD25J4g0JSJ7Aq8CV6vqmrDjiTdV3aaq7bDdCTuJSEp3J4pID2CZqk4JO5YK1kVVOwAnA5cHXcipLAvoADyqqu2B34ChsXpyTxBpKOiHfxUYp6qvhR1PRQqa3xOB7iGHEm9dgJ5Bn/xLwHEiMjbckOJPVZcEP5cBrwOdwo0o7nKB3IgW8XgsYcSEJ4g0EwzYPgX8oKr3hx1PRRCReiJSK7heDTgBmBVuVPGlqjeqaiNVbYJt9/uxqp4fclhxJSJ7BBMvCLpZugEpPTtRVZcCi0WkZXDoeCBmE07ituVoshCRF4FjgLoikgvcqqpPhRtVXHUBLgCmB33yAMNU9d0QY4q3BsBzIpKJfSl6RVXTYtpnmqkPvG7fgcgCXlDV98INqUJcAYwLZjDNBy6K1ROn/TRX55xz0XkXk3POuag8QTjnnIvKE4RzzrmoPEE455yLyhOEc865qDxBOLcLIrItqA5acInZSlURaZIulYRd8kn7dRDOlcKGoEyHc2nFWxDO7aZg74F7gr0mvhaRA4Pj2SLyHxH5LvjZODheX0ReD/al+FZE/hA8VaaIPBHsVfFBsNobEblSRGYGz/NSSG/TpTFPEM7tWrUiXUznRty3RlU7AY9g1VMJrv9DVQ8BxgEPB8cfBj5R1UOxejkzguPNgdGq2hpYBZwZHB8KtA+eZ0C83pxzxfGV1M7tgoisU9U9oxxfABynqvODAohLVbWOiCzHNmXaEhz/RVXrikge0EhVN0U8RxOs/Hjz4PYQoJKqjhSR97DNrN4A3ojY08K5CuEtCOfKR4u5Xtw50WyKuL6NwrHBU4HRQEdgioj4mKGrUJ4gnCufcyN+fhFc/x9WQRWgL7bFKcB/gIHw+wZGNYp7UhHJAPZX1QnYpj+1gJ1aMc7Fk38jcW7XqkVUvgV4T1ULprpWEZGvsC9bfYJjVwJPi8j12G5fBdU1rwLGiMglWEthIPBLMa+ZCYwVkZqAAA/EcitJ50rDxyCc203BGESOqi4POxbn4sG7mJxzzkXlLQjnnHNReQvCOedcVJ4gnHPOReUJwjnnXFSeIJxzzkXlCcI551xU/w+JnWCztJfeLwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU9dXA8e9hB0FFNhFIgoqKW0FT0GIFxQVXXKhFqYqtoBUr9dUKinVBsZT6iq+KKCJuoGixAi5oRVlUBAkKyCKCyBJBCasgezjvH+eGTMJkn5vJZM7neebJzJ07M78JZM78tnNEVXHOOefyqxLvBjjnnKuYPEA455yLygOEc865qDxAOOeci8oDhHPOuag8QDjnnIvKA4QrNyJSVUS2iUhKLM+NJxE5WkRivlZcRM4RkRURt5eIyG+Lc24pXmukiNxT2se7yqtavBvgKi4R2RZxsw6wC8gObt+kqmNK8nyqmg3UjfW5yUBVj43F84jIjcAfVLVTxHPfGIvndpWPBwhXIFXd/wEdfEO9UVUnF3S+iFRT1b3l0TbnXPh8iMmVmog8LCKvi8hrIrIV+IOInC4iM0Vks4isFZEnRKR6cH41EVERSQtujw7unyQiW0XkcxFpWdJzg/svEJFvRWSLiDwpIp+JSM8C2l2cNt4kIstEZJOIPBHx2KoiMlRENojId0CXQn4/94rI2HzHhonIY8H1G0VkcfB+vgu+3Rf0XJki0im4XkdEXgnathA4NcrrLg+ed6GIXBocPwl4CvhtMHy3PuJ3+0DE428O3vsGERkvIk2L87sp4e/5gKE5Efk08t8seJ1vgvexQER+VdBruZCoql/8UuQFWAGck+/Yw8Bu4BLsy0Zt4NdAe6x3eiTwLXBrcH41QIG04PZoYD2QDlQHXgdGl+LcxsBWoGtw3/8Ae4CeBbyX4rRxAnAIkAZszHnvwK3AQqA50ACYbn9GUV/nSGAbcFDEc68D0oPblwTnCHA2sAM4ObjvHGBFxHNlAp2C648CU4H6QCqwKN+5VwFNg3+Ta4I2NAnuuxGYmq+do4EHguvnBW1sA9QCngY+Ls7vpoS/56Pz/96AT3P+zYCrgdVY8BPgGKBFvP8Oku3iPQhXVp+q6tuquk9Vd6jqbFWdpap7VXU5MALoWMjjx6lqhqruAcZgH0wlPfdiYK6qTgjuG4oFk6iK2cZ/qOoWVV2BfRjnvNZVwFBVzVTVDcDgQl5nObAAC1wA5wKbVTUjuP9tVV2u5mPgIyDqRHQ+VwEPq+omVV2J9QoiX/cNVV0b/Ju8igX39GI8L0APYKSqzlXVnUB/oKOINI84p6DfTR6l+L8Q6UZgsKrOCX4/36rq6mI+1sWIBwhXVnn+aEXkOBF5V0R+FJGfgYFAw0Ie/2PE9e0UPjFd0LlHRLZDVRX7xh1VMdtYrNcCVhbSXoBXsW/DYN/m90/si8jFIjJLRDaKyGbs23thv6scTQtrg4j0FJF5wdDOZuC4Yj4v2Pvb/3yq+jOwCWgWcU6x/s1K8X8hUgvgu2Ke60LiAcKVVf4lns9i35qPVtWDgfuwIYIwrcWGfAAQESHvB1p+ZWnjWuzDK0dRy3BfB84JvoF3xQIGIlIbGAf8Axv+ORT4bzHb8WNBbRCRI4HhwJ+BBsHzfhPxvEUtyV2DDVvlPF89bCjrh2K0K7/Cfs+/BM9fJ+L8wyOurwaOKsVruhjyAOFirR6wBfhFRFoDN5XDa74DnCIil4hINaAv0CikNr4B/FVEmolIA6BfYSer6k/Y2PoLwBJVXRrcVROoAWQB2SJyMdC5BG24R0QOFdsncmvEfXWxIJCFxcobsR5Ejp+A5jmTxVG8BvxJRE4WkZpYAPtEVQvskRWisN/zj8HlD8HEf28iAhMwErhLRNqKaSUikUHRlQMPEC7W7gCuxyaNn8W+QYcq+BD+PfAYsAH75vkVtm8j1m0cjs0VfA3MxnoBRXkVm3R+NaLNm4Hbgbewid5uWKArjvuxnswKYBLwcsTzzgeeAL4IzjkOmBXx2A+BpcBPIhI5VJTz+PexoaC3gsenYPMSpVHg7zkYBuwF3IPNFx0d2U5VfQ34Z/CYn4H/YD0ZV47E/p2cqzxEpCo2VNJNVT+Jd3ucS1Teg3CVgoh0EZFDgmGRvwN7sW/RzrlS8gDhKoszgOXYcEUX4DJVLWiIyTlXDD7E5JxzLirvQTjnnIuq0iTra9iwoaalpcW7Gc45l1DmzJmzXlWjLguvNAEiLS2NjIyMeDfDOecSiogUmA3Ah5icc85F5QHCOedcVKEGiGBt+pIgd3z/KPcPFZG5weXbILFYzn3ZEfdNDLOdzjnnDhTaHESwm3UYluI4E5gtIhNVdVHOOap6e8T5fwHaRjzFDlUtLPVzkfbs2UNmZiY7d+4sy9MkvVq1atG8eXOqVy8ofY9zrjIKc5K6HbAsyANPUFmrK1bcJJqrsRwzMZOZmUm9evVIS0vDEny6klJVNmzYQGZmJi1btiz6Ac65SiPMIaZm5M1Zn0kBKZhFJBVoCXwccbiWiGQEJQsvK00Ddu7cSYMGDTw4lIGI0KBBA++FOVcBjRkDaWlQpYr9HDOmqEeUTJg9iGifygVt2+6OVQvLjjiWoqprgvz2H4vI16qap4BIkCK4N0BKSvS0/B4cys5/h85VPGPGQO/esH273V650m4D9Cht/t18wuxBZJK3qElzLMNmNN2xPPT7qeqa4OdyrKxh2/wPUtURqpququmNGhWW/t855yqXAQNyg0OO7dvteKyEGSBmA61EpKWI1MCCwAGrkUTkWCzP++cRx+oHWTkRkYZABwqeu3DOuaShCrNmWY8hmlWrYvdaoQUIVd2LVbr6AFgMvKGqC0VkoIhcGnHq1cBYzZs1sDWQISLzgClY8fLQA0QY43mbN2/m6aefLvHjLrzwQjZv3lz0ic65pNK1K5x2GhQ08lvAaHupVJpsrunp6Zo/1cbixYtp3bp1sR6ffzwPoE4dGDGibON5K1as4OKLL2bBggV5jmdnZ1O1atXSP3E5K8nv0jkXG3v2wHvvwdix8MILUKuWfVbt2GH39+1b9s8sEZmjqunR7qs0uZiKo1OnA49ddRXccgvcfXf08by+fe2XvX49dOuW9/6pU4t+zf79+/Pdd9/Rpk0bqlevTt26dWnatClz585l0aJFXHbZZaxevZqdO3fSt29fegezTDm5pbZt28YFF1zAGWecwYwZM2jWrBkTJkygdu3aUV/vueeeY8SIEezevZujjz6aV155hTp16tCzZ08uvvhiugVvom7dumzbtg2AIUOG8Morr1ClShUuuOACBg8eXPQbc86FZtEiCwgvvwzr1kGTJrBkCfzqV3k//GvXtjmHVaus5zBoUOwmqMFTbeyXWUBJ9g0byva8gwcP5qijjmLu3Ln861//4osvvmDQoEEsWmQjZqNGjWLOnDlkZGTwxBNPsCHKCy5dupQ+ffqwcOFCDj30UN58880CX++KK65g9uzZzJs3j9atW/P8888X2r5JkyYxfvx4Zs2axbx587jrrrvK9oadc2Uybx6ccAI8/jh06AATJ8Lq1RYc8uvRA1asgH377GcsgwMkWQ+isG/8KSnRJ31SU+1nw4bF6zEUpV27dnk2nD3xxBO89dZbAKxevZqlS5fSoEGDPI9p2bIlbdrYpvJTTz2VFStWFPj8CxYs4N5772Xz5s1s27aN888/v9D2TJ48mRtuuIE6deoAcNhhh5XmbTnnSmHfPpg2DUaNgkaN4LHH4OSTbZioa1do3Di+7fMeRGDQIBu/i1Snjh2PpYMOOmj/9alTpzJ58mQ+//xz5s2bR9u2baNuSKtZs+b+61WrVmXv3r0FPn/Pnj156qmn+Prrr7n//vv3P1+1atXYt28fYLujd+/evf+673NwrnytWgUPPQRHHw1nnw1vvw05mWxEoFev+AcH8ACxX48eFrVTU+0fKDW17BPUAPXq1WPr1q1R79uyZQv169enTp06fPPNN8ycObNsLwZs3bqVpk2bsmfPHsZELMNKS0tjzpw5AEyYMIE9e/YAcN555zFq1Ci2BxMwGzduLHMbnHMH2rnTlqgCDB4M990HRx5pk85r18I//xnf9kWTVENMRenRI/ZjeA0aNKBDhw6ceOKJ1K5dmyZNmuy/r0uXLjzzzDOcfPLJHHvssZx22mllfr2HHnqI9u3bk5qaykknnbQ/OPXq1YuuXbvSrl07OnfuvL8n06VLF+bOnUt6ejo1atTgwgsv5JFHHilzO5xzFhC+/NKGkF59Fd55x+YV+veHu+6y5fQVmS9zdcXiv0vnim/7dhuBGDUKvv7alqdeeaUFhhNPjHfr8ipsmasPMTnnXAzs3QvfBdniqla1OYbatWH4cBtCGj264gWHovgQU4Lq06cPn332WZ5jffv25YYbbohTi5xLTkuW5O5ZqF0bli6FmjVh8eKKMdFcFh4gEtSwYcPi3QTnktrkyfDAA/DZZ9ZjuPBC+OMfcyeiEz04gAcI55wrFlX49FM46ig44gjYts020g4ZAtdeC4cfHu8Wxp7PQTjnXCEyM+GRR+CYY+DMM+G55+z4pZdaSoy//a1yBgfwHoRzzkW1bx9cfrktTd23Dzp2hL//3VYjgWV9ruw8QDjnXGDuXJgyBW6/3QJAixZwzz3Qs6cNLSWbJIiBJRB2gddiqFu3brm/pnPJbMMGePJJaNvWLvfcYxlUAZ56yparJmNwAA8QuXIKQqxcabNROQVe4xAknHPlY9Ikm3C+7Tb7XvjUU/DDD5VjBVIsJM8Q01//av3HgsycCbt25T22fTv86U+5s1L5tWljOXkL0a9fP1JTU7nlllsAeOCBBxARpk+fzqZNm9izZw8PP/wwXbt2LfItbNu2ja5dux7wuPxFiR599FG2bdvGAw88wLJly7j55pvJysqiatWq/Pvf/+aoZP065JLesmXw4ouWOvt3v4N27eDPf7YhpCBhsovgPYgc+YNDUceLqXv37rz++uv7b7/xxhvccMMNvPXWW3z55ZdMmTKFO+64g+KkPKlVq1aJH9ejRw/69OnDvHnzmDFjBk2bNi3T+3GuoipohHjbNgsKHTtCq1bwj39ATlaeBg3sO54Hh+iSpwdRxDd90tIKLghRhkIQbdu2Zd26daxZs4asrCzq169P06ZNuf3225k+fTpVqlThhx9+4KeffuLwItbKqSr33HPPAY8ryNatW/nhhx+4/PLLAQswzlVG+UsG54wQA4wcaX/CrVrZctXrroNmzeLW1ISSPAGiKIMGRS9KHYOCEN26dWPcuHH8+OOPdO/enTFjxpCVlcWcOXOoXr06aWlpUetA5FfQ4yJrPQD7n6uyJGJ0rigDBkQvGTxggKXBeOghy6LqpU9KJtQhJhHpIiJLRGSZiPSPcv9QEZkbXL4Vkc0R910vIkuDy/VhthMIryAENsw0duxYxo0bR7du3diyZQuNGzemevXqTJkyhZXRei5RFPS4Jk2asG7dOjZs2MCuXbt45513ADj44INp3rw548ePB2DXrl376z44V1ksXRq98w9WmOess+CMMzw4lEZoPQgRqQoMA84FMoHZIjJRVRflnKOqt0ec/xegbXD9MOB+IB1QYE7w2E1htRcIpyAEcMIJJ7B161aaNWtG06ZN6dGjB5dccgnp6em0adOG4447rpjNi/646tWrc99999G+fXtatmyZ5/leeeUVbrrpJu677z6qV6/Ov//9b4488siYv0fnysOWLTZc9PHHcM45cMkltuhQJDcHUqSUlHJvYuWiqqFcgNOBDyJu3w3cXcj5M4Bzg+tXA89G3PcscHVhr3fqqadqfosWLTrgmCsd/126eNm7V/Xuu1XbtVOtUkUVVGvXVv3nP+3+fftUX35ZtU4duy/nUqeO6ujR8W17IgAytIDP1TDnIJoBqyNuZwLto50oIqlAS+DjQh7r00rOVXJ798KcOfDRR3b9vvssU+rbb8PBB9smts6d4fTTLaU2WO/h2mtt9dKAATaslJJi04chDAgklTADRLQRv4JmTbsD41Q1uySPFZHeQG+AlErUl/z666+59tpr8xyrWbMms2bNilOLnAvXG2/YSqSpU+Hnn+3YWWdZgAD46iuoVsSnVUgjxEktzACRCbSIuN0cWFPAud2BPvke2ynfY6fmf5CqjgBGgJUcjfbEqook2OzUSSedxNzCNvWVM/XVUC6GVqywHsK0abYHtWZN25ewcCF07249hLPOgkaNch9TVHBw4Qjz1z4baCUiLYEfsCBwTf6TRORYoD7wecThD4BHRKR+cPs8bA6jRGrVqsWGDRto0KBBwgWJikJV2bBhg++hcGWyYAH83/9ZYPj+ezvWpIldP+44Gw4aMiS+bXQHCi1AqOpeEbkV+7CvCoxS1YUiMhCbFJkYnHo1MFYjvqaq6kYReQgLMgADVXVjSdvQvHlzMjMzycrKKtubSXK1atWiefPm8W6GSxBbt1rv4OOP4YorbInp5s02jNSpk2W96dwZjj8+d+lp9epxbbIrgFSW4YP09HTNyNk/75wrV9u3WwqLjz6CL76A7GyoVQsee8xyHWVn29oiHyqqeERkjqqmR7vP/7mccyWSnQ1ffmnBoE4dy4Raq5bNJ7RsCf37w9lnw29+Y8fBViK5xOMBwjlXLGPGwLhxttJoc5Dz4OKLc1Nlr1yZu/TUVQ4eIJxzB1i9One4aNgwmyuYPNky5l95pc0hnH22TTTn8OBQ+XiAcM4BNmw0YoQFhmXL7FjjxnDvvVZUZ/jw3CEjlxy8HoRzlVBR1XO3bbNqanfeaUtQwYaIXn3Vlp0OHQrz58OPP1pwAA8Oych7EM5VMgXVRvjlFyun+dFHMGuWpbKoUcOK5Zx4os0nbNzoK41cLl/m6lwlU1DtqxYtYP16CwY5cwgdOthKJJe8fJmrc0niuecKro2QmWmb2A46qHzb5BKXBwjnEpAqLF5su5WXLcutqDt+vA0R7d174GNSUjw4uJLxSWrnEshHH1lCu6ZN4YQT4C9/saCQM98wbhy8+OKBw0Yxqp7rkowHCOcqqO+/h1GjrNbBihV27Lvv4JNPrJrayJF2+/vvcwNC7dqhVs91ScYnqZ2rQFasgIEDbegoZy6hcWMYO9ZSYO/da2krPDmxixWfpHauAlq7FqZMsUvHjvCHP9hu5IkT7fadd9pKo9atcwOCL0F15cn/uzlXjlShb1/48EP45hs7dsghluQObG5h3Trb4OZcvHmAcC4kmzZZXYQpU2DXLnjmGesJLFxoAeGPf7Rho7Zt82Y79eDgKgoPEM7F2IgRFgzmzrUeQ+3aNqmsmpv0zucQXCLwAOFcKf3yC3z6qfUQpk2D99+34aKff7afDzxgPYR27fJmOvXg4BKFBwjnSmjmTLjjDkuFvXevTRy3bw8//WSB4c477eJcovMA4VwBdu2ypHY5K41uvRW6dYO6da2q2p13Wg+hQwffoewqJw8QzuXzyy/QtSvMmAE7dtiQUNu2NocAluxu5sz4ttG58hDqegkR6SIiS0RkmYj0L+Ccq0RkkYgsFJFXI45ni8jc4DIxzHa6yq2g2gjZ2TB7NgwZAhdcAL162fGDDrI5g969LY3Fhg0wZw787nfxegfOxUdoPQgRqQoMA84FMoHZIjJRVRdFnNMKuBvooKqbRKRxxFPsUNU2YbXPJYeCaiOMHg2ffw5bttjx1q0hPWIv6bvvln9bnatowhxiagcsU9XlACIyFugKLIo4pxcwTFU3AajquhDb45LQgAG5wSHH9u02RHTVVbZTuVMnOPzwuDTPuQotzADRDFgdcTsTaJ/vnGMAROQzoCrwgKq+H9xXS0QygL3AYFUdH2JbXSW1alX041u22H4F51zBwgwQ0VZ7588MWA1oBXQCmgOfiMiJqroZSFHVNSJyJPCxiHytqt/leQGR3kBvgJSUlFi33yW4PXssy+kvvxx4n/93ca5oYU5SZwItIm43B9ZEOWeCqu5R1e+BJVjAQFXXBD+XA1OBtvlfQFVHqGq6qqY3atQo9u/AJbR+/Sw4VK+e97jXRnCueMIMELOBViLSUkRqAN2B/KuRxgNnAYhIQ2zIabmI1BeRmhHHO5B37sK5IvXrB6+9Bi+84LURnCuN0IaYVHWviNwKfIDNL4xS1YUiMhDIUNWJwX3nicgiIBv4m6puEJHfAM+KyD4siA2OXP3kXEFmz4annoLnn4cmTaz6GnhAcK40vGCQqzTGj4drrrHAMH06tGhR9GOcS3aFFQzyxMIu4anC44/DFVfASSfZElYPDs6VnQcIl/AefBBuvx0uv9xyJjVpEu8WOVc5eC4ml/AuvRR274aHH/ZiO87Fkv85uYS0Zg08+aRdP+UUeOQRDw7OxZr3IFzCmT8fLrrISnp27eqb3pwLi3/ncgnlgw/gjDNg3z6r5ubBwbnweIBwCWPkSOs5HHmkFfJp47l+nQuVBwiXMOrWhfPPh08+gebN490a5yo/DxCuQtuxA6ZOtevdu8M770C9enFtknNJwwOEq7CysqxeQ5cutmoJLJ+Sc658+ComVyEtWQIXXmiBYcwYOOKIeLfIueTjAcJVONOnw2WXQbVqNrzUPn+ZKedcufAA4SqcDz+0dBnvvQctW8a7Nc4lL5+DcBWCKqwOCtQ++KAtY/Xg4Fx8eYBwcbd7N9xwA6Snw08/WcqMgw+Od6uccx4gXFxt2mR7G156Cfr0gcaN490i51wOn4NwcbN8ue2MXr4cXnkF/vCHeLfIORfJA4SLm4cesiGl//4XOnaMd2ucc/n5EJMrd3v22M8nn7TJaA8OzlVMHiBcuVGF//1fOO002LrVciu1ahXvVjnnChJqgBCRLiKyRESWiUj/As65SkQWichCEXk14vj1IrI0uFwfZjtd+PbuhVtugTvvhKOOsk1wzrmKLbQ/UxGpCgwDzgUygdkiMlFVF0Wc0wq4G+igqptEpHFw/DDgfiAdUGBO8NhNYbXXhWfrVvj972HSJOjXz6u/OZcowvwzbQcsU9XlqrobGAt0zXdOL2BYzge/qq4Ljp8PfKiqG4P7PgS6hNhWF6JevWwiesQIGDzYg4NziSLMP9VmwOqI25nBsUjHAMeIyGciMlNEupTgsS5BDB5saTN69Yp3S5xzJRFmgIiWmFnz3a4GtAI6AVcDI0Xk0GI+FhHpLSIZIpKRlZVVxua6WHrvPfjTn6w0aFoanHdevFvknCupMANEJtAi4nZzYE2Ucyao6h5V/R5YggWM4jwWVR2hqumqmt6oUaOYNt6V3tNPwyWXwFdfwZYt8W6Nc660wgwQs4FWItJSRGoA3YGJ+c4ZD5wFICINsSGn5cAHwHkiUl9E6gPnBcdcBbZvn61S6tPHajlMnw7168e7Vc650gptFZOq7hWRW7EP9qrAKFVdKCIDgQxVnUhuIFgEZAN/U9UNACLyEBZkAAaq6saw2upio3dveP55uPVWePxxqFo13i1yzpWFqB4wtJ+Q0tPTNSMjI97NSGrTp9uwUt++8W6Jc664RGSOqqZHu88XHLoyWbQIhg+362ee6cHBucrE97O6Uvv4Y7jiCqhdG66+Gg49NN4tcs7FkvcgXKm8+KLVcWjeHGbO9ODgXGVUrAAhIqeJSL2I2/VExEvJJ6kHH7QKcJ06waefQmpqvFvknAtDcXsQw4FtEbd/CY65JNSkCfzxj7YZznsOzlVexQ0QohHLnVR1Hz5/kVQ2brRVSgA33wwjR0L16vFtk3MuXMUNEMtF5DYRqR5c+mIb2lwSWLYMTj8dLr/cMrMCSLRkKM65SqW4AeJm4DfAD1gajPZA77Aa5SqOGTMsOGzYABMmQL16RT/GOVc5FGuYKEjD3T3ktrgK5o034LrroEULm2/w6m/OJZdiBQgReYEo2VRV9Y8xb5GrMKZOhfR0GD8eGjaMd2ucc+WtuBPN70RcrwVcTpTsqi7x7dkDa9dCSgo88YSVCq1VK96tcs7FQ3GHmN6MvC0irwGTQ2mRi5stW+Cqq2DJEli4EA46yGtHO5fMSvvn3wpIiWVDXHytWgUXXQTffAPPPmvBwTmX3Io7B7GV3DkIBX4C7gqrUa58zZljBX5++QUmTYJzzol3i5xzFUFxh5jqichhWM8hZ0S6cuQJT0JjxsCAAdZrSEmx3dDVq9uS1hNOiHfrnHMVRXF7EDcCfbHSn3OB04DPgbPDa5oLw5gxVthn+3a7vXIlrFsHjz7qwcE5l1dxN8r1BX4NrFTVs4C2QFZorXKhGTAgNzjk2LEDhgyJT3uccxVXcQPETlXdCSAiNVX1G+DY8JrlwrBtm/UYolm1qnzb4pyr+IobIDJF5FBgPPChiEzA90EklK++Kjwtd4qvSXPO5VPcSerLg6sPiMgU4BDg/dBa5WJi/XpLtHfaaXD88bZSqWVLG06KHGaqUwcGDYpfO51zFVOJK8qp6jRVnaiqu4s6V0S6iMgSEVkmIv2j3N9TRLJEZG5wuTHivuyI4xNL2s5ktnYt3HGH9Ri6d4fsbKhZ06rA3X8/jBhh94nYzxEjoEePeLfaOVfRhLZPVkSqAsOAc7EMsLNFZKKqLsp36uuqemuUp9ihqm3Cal9ltHo1DB4Mzz9vKTOuuQbuvhuqVs17Xo8eHhCcc0ULsyZ1O2CZqi4Pehtjga4hvl7SyinlNG8ePPecZWD99lt45RUbWnLOudIIM0A0A1ZH3M4MjuV3pYjMF5FxItIi4ngtEckQkZkiclmI7UxYCxdaT+C+++z2RRfB99/bkNFRR8W3bc65xBdmgIhWcyz/7uu3gTRVPRlL/vdSxH0pqpoOXAM8LiIHfOSJSO8giGRkZSXPtow5c+CKK+DEE62IT07pTxFoFi0EO+dcKYQZIDKByB5Bc/ItjVXVDaq6K7j5HHBqxH1rgp/LganY5jzyPX6EqqaranqjRo1i2/oK6pFHrEbDxx9bz2HlytwehHPOxVKYyZxnA61EpCVWqrQ71hvYT0Saqura4OalwOLgeH1gu6ruEpGGQAcgKff6qsKUKbba6Kij4MILradwyy1wyCHxbp1zrjILrQehqnuBW4EPsA/+N1R1oYgMFJFLg9NuE5GFIjIPuA3oGRxvDWQEx8ysLM8AABmDSURBVKcAg6OsfqrUVOHdd6FDB+jcGYYOteNt2tjKJA8OzrmwiWrlSMqanp6uGRkZ8W5GTEycCA88YLufU1Kgf3+44Qav7Oaciz0RmRPM9x7A64VVENnZUKWKDR9Nnmx5k154wVYp5UxCO+dceQpzktoVw+7dMHIkHHMMTJ9uxx55BBYvhp49PTg45+LHA0Sc7NgBTz0FRx8NvXpB/fq5O57r1j1w97NzzpU3H2KKA1VLoDd/vk1CjxgB559vw0vOOVdReIAoJ5s3W7K8W2+FatXg3nuhcWM480wPDM65iskDRMjWr4fHH4cnn4Sff4aTT4azz4bf/S7eLXPOucL5HERItm+HO++0DW6PPALnnQdffmnBwTnnEoH3IGJs+3YrwFOzJnzwAVx5pW1sa9063i1zzrmS8QARI0uXWi2G996zVNv16kFGhgUK55xLRD7EVEYLFlhhnuOOg1dftbmFPXvsvgobHMaMgbQ025mXlma3nXMuH+9BlME338BJJ8FBB1mJz//5Hzj88Hi3qghjxkDv3rlFqVeutNvgZeacc3l4D6KEZsyA4cPt+nHHWQW3lSthyJAECA4AAwbkBocc27fbceeci+ABohhUrf7C2WfbxrZBg2DnTrvvxhuhQYP4tq9EVq2KfnzlSlixolyb4pyr2DxAFGHu3NyU2998A489BkuWJGhm1fXrbZdeQVq2tCLWd9xhGQN37Sr4XOdcpedzEFHs22c7nw87zPIirVsHTz+d4Cm316yBc8+17lDNmnk//OvUgQcftODx3nuWJOqxx2xypXNnuOACu6Smxq/9zrlyl/T1IMaMseH3VaugRQur2DZtmiXRmzjRztm3zxb8JKwVK+yDft06ePtt+OGH3DedkmJjZpET1L/8YmXs3nsPJk3KHXo6/ngLFBdeCGecATVqxOPdOOdiqLB6EEkdIPIv6MnRvDn861/QvXsMGxgvS5bAOedYgYn334f27Uv2eFV7jpxgMX265SivWzdv7yIlJZz2O+dC5QGiAGlpNjebX0pK9OMJZ/58G1YC+PBDSwRVVtu25e1d5PyiTjghN1h478K5hOEBogBVqtgX5PxEbFgpoc2aBV262Df9yZPh2GNj/xqqNnM/aZIFjOnTbZdg3brWa8kJGC1axP61nXMx4SVHC1BQTyHhR0umToVLLrF84h99ZF2lMIhYkqnWrW2X4LZtth44p3cxfrydd+KJucGiQwfvXTiXIEKdehWRLiKyRESWiUj/KPf3FJEsEZkbXG6MuO96EVkaXK4Po32DBtkCnkh16tjxhDVpUu6cwCefhBccoqlbFy69FJ55xia2Fy60yZzGjS3n+dlnQ8OGcMUVtsMwM7P82uacKzlVDeUCVAW+A44EagDzgOPzndMTeCrKYw8Dlgc/6wfX6xf2eqeeeqqWxujRqqmpqiL2c/ToUj1NxTBunGr16qqnnKKalRXv1uT188+q48er9u6t2qKFqg1QqZ50kupdd6lOmaK6e3e8W+lc0gEytIDP1TB7EO2AZaq6XFV3A2OBrsV87PnAh6q6UVU3AR8CXcJoZI8e9mV33z77mbDpiF5+Ga66Cn79axvmadgw3i3Kq1496NoVnn3WxvUWLLD8JA0b2p6Ls86y61deCSNH2lJc51xchRkgmgGrI25nBsfyu1JE5ovIOBHJmc0s1mNFpLeIZIhIRlZWVqzanXiefhquv94+ZP/7XzjkkHi3qHAiturpb3+zYLZhA/znP/D738MXX0CvXrbW+Fe/gv79bWNKTopc51y5CTNARKu0nH/N0NtAmqqeDEwGXirBY1HVEaqarqrpjRo1KlNjE9aQIdCnj01Kv/OO7X5ONAcfDJdfDiNG2Oa9r7+Gf/7TtrL/7/9Cp07Wu+jWDZ5/3naFO+dCF2aAyAQi1zc2B/L8ZavqBlXNyfnwHHBqcR+b9FTh73+Hfv1sR9+bbyZwHpAIIrbq6a67bL9FTu/iqqtg5kzLjtisGbRpY6X6cpbWOudiLrR9ECJSDfgW6Az8AMwGrlHVhRHnNFXVtcH1y4F+qnqaiBwGzAFOCU79EjhVVTcW9HqlTbWRkFRtWenjj8Of/mTj+lWrxrtV4VO1uYucZbSffQZ799qQ2rnn2uqtLl3giCPi3VLnEkZh+yBC60Go6l7gVuADYDHwhqouFJGBInJpcNptIrJQROYBt2GrmggCwUNYUJkNDCwsOCSV7GzLD/L449C3ry0XTYbgANa7OOkk6zVNnWrZaceNs6GnGTMsWDZrBm3bwj332DLfvXu9gp5zpZTUO6kTzp49cN11MHYs3HsvDBxoH5rOehfz51vPIqd3kZ0NtWtb7qjs7Nxz69Sx+Y6EXbLmXOx4qo3KYOdOW+UzcaJN4N51V7xbVLFt3mwpRnr2tOy0+aWmeoEk54jTEJOLoV9+sVVKEyfCsGEeHIrj0ENt6Cl/qt4cK1fmlgV0zkXlAaKi27IFzj/f9gu8+CLccku8W5RYCkus1bo1vP569IyNziWCkOfXPEBUZOvXW/6iL76wD7LrQ0lJVbkVlHCrf3/bf9G9uyUQnDkzPu1zrrRyCtqsXGlfclautNsxDBIeICqqNWugY0dYtAgmTLDhEldyPXrYhHRqqk3op6ba7X/8A7780tJ6fP89nH46XH21z0u4xDFgwIFDqNu32/EY8UnqimjFCqun8NNPViK0U6d4t6hy27bNdqQ/+qgl5br9dtuEd/DB8W6ZcwWLUUEbn6ROJEuWwG9/azuIJ0/24FAe6ta1JcNLltiO7cGDrSj5M8/YPgrnKhJVeOKJgufOYljQxgNERTJ/Ppx5JuzaZRvBSlo/2pVNixaWFXf2bDjuOPjzny1h4Pvvx7tlzpmNGy1vWd++lm6mdu2898e4oI0HiIriiy+st1Cjhu0A/tWv4t2i5JWebhlk33zTgnVOCo8FC+LdMpfMZsywoPDeezB0qM2hPffcgfNrMdwA6gGiIpg2DTp3hvr1LTiEUT/alYyIVb5btMjqVcyaZUH7pptsbsi58rJvny2qOPNMqF7dAsVf/2r/R0MuaOMBIt7ef9++ncajRKgrWo0aNmm9bBnceiuMGmXzE//4B+zYEe/Wucrup5+sB3vPPbaS8csvrYdbTjxAxNObb1oN59atrRfhWUgrrgYN4P/+z+psd+5sf7DHHQevveYb7Vw4PvrIhpSmT7eho9deK/diYB4g4qWilwh10R1zDIwfb/9mhx0G11xjeyhmzIh3y1xlsXev1Xo591wbds6pshiHxJweIOJh+PC8JUIPPTTeLXIlddZZkJEBL7xgVfA6dLCAv3x5vFvmEllmpmVPePhhuOEGW1F30klxa44HiPL2r39ZPqVELhHqTNWqli126VK4/354910bLrzrLssm61xJvP22LYT46isYPdrK68b588EDRHlRhfvusw+P3/++8pQIdfZH/MAD8O23NuT06KPQqhU8/bRvtHNF273bKkReeqktVpkzp8LUKvEAUR5U4Y474KGHrOrZmDG2XM1VLs2a2ZBTRobV1e7TB04+2XoWPpHtovnuOxueHDrUVsl9/rnNc1UQHiDCllMidOhQ2/04YkTylAhNVqecYpPY48dbD+Lii+G882ynvHM53njD/q8sWwb/+Q88+WSFG1XwABGmPXvg2mstY+i991qQqOK/8qQgAl272u7rxx+3YYM2bWw1yo8/xrt1Lp527ICbb7ah5uOPh7lzLX1GBeSfVmHZuRN+9ztbuzx4sA0vef3o5FOjhvUcly2z3a8vvWQb7R5+uOBqd67yWrzYcqw9+yz062d7HFJT492qAoUaIESki4gsEZFlItK/kPO6iYiKSHpwO01EdojI3ODyTJjtjLmcEqETJliJ0H794t0iF2+HHWYpOxYutOGmv//dUqqMHl2i1MwuQalaRcj0dOtBTppkXxwr+FxkaAFCRKoCw4ALgOOBq0Xk+Cjn1QNuA2blu+s7VW0TXG4Oq50x5yVCXWFatbLx5qlToXFjG4Js397SrLjKaetWuO4629fQvr0NKXXpEu9WFUuYPYh2wDJVXa6qu4GxQNco5z0EDAESv4K8lwh1xdWxo22CevllWLvWErF162arWlzlMXcunHoqvPoqPPggfPhhQqXUCTNANANWR9zODI7tJyJtgRaq+k6Ux7cUka9EZJqI/DbaC4hIbxHJEJGMrKysmDW8VNauzS0ROn68lwh1RatSxXoQ335rBYsmTbKNdnfcAZs2xbt1rixUbXi5fXsbcv74Y9sHlWArGMMMENFmZPcvBheRKsBQ4I4o560FUlS1LfA/wKsickD9R1UdoarpqpreqFGjGDW7FFautCpwq1bZH/mFF8avLS7x1KljcxJLl1rAGDrUJrKffNJWwrnEsmkTXHml7Ws45xzrRXTsGO9WlUqYASITaBFxuzmwJuJ2PeBEYKqIrABOAyaKSLqq7lLVDQCqOgf4Dqg4u0ciffstnHGGlwh1ZXfEEZZe4csvbUnsbbfZhruJE32jXaKYORPatrW0GY8+aj/j+eW1jMIMELOBViLSUkRqAN2BiTl3quoWVW2oqmmqmgbMBC5V1QwRaRRMciMiRwKtgIqXBc1LhLowtGljXzYmTszdT9G5s30TdRXTvn0wZIiNJFSpAp99ZkOFCb7vKbTWq+pe4FbgA2Ax8IaqLhSRgSJyaREPPxOYLyLzgHHAzaq6May2lkpOidBq1bxEqIs9EVsq/fXXNtQ0f77tuv3jH2HNmqIf78pPVhZcdJEtZ7/sMusBtmsX71bFhGgl6bqmp6drRkZG+bzYtGmWPqFxY/um17Jl+byuS16bNlkx+ieesLXz/frZN1TPBhxfU6dagsaNG23H/E03JdyGWBGZo6pRy9Qldv8nHnJKhLZoYT0HDw6uPNSvb2PaixfbIoj777eNdi+/7Bvt4iE72/4Nzj4bDj7YapbffHPCBYeieIAoCS8R6uLtqKPg3/+2LydHHGF7bX79a/sm68rHDz/YnNDAgbYBLiOj0g4xe4AorldeyVsiNIFXJrhK4IwzbMXM6NGwbp1VuLv8clsq68Lz3nu2iCAjw/Jqvfgi1K0b71aFxgNEcTzzjH1TOOss+OADLxHqKoYqVaywzJIllvxv8mTLDnr77ZZWPi3NzklLsxokrvR274a//c0mo484wrLzXnddvFsVPlWtFJdTTz1VQzFkiCqoXnKJ6o4d4byGc7Gwdq1qr172/zX/pU4d1dGj493CxLR8uWq7dvZ7vOWWSvc5AGRoAZ+r3oMoiKpNQnmJUJcoDj/ceg5Nmx543/btnlW4NN580za+LVkC48ZZ+owk+hzwABFNTonQgQO9RKhLPAUVJPrhBzjySNtL8dJLsGJFuTYroezcaZmYu3Wz1WJffWXpM5KMB4j8srNtLfPQoZbqwEuEukSTkhL9eP36ttpmwgTo2dOWaKel2UqoUaMsk2wl2RdVJkuWwGmnwfDhcOedSb2cvVq8G1Ch7NljfzivvgoDBngVOJeYBg2yOuiRFevq1LEd2T162L6JBQtsqfa0abYy5+WX7bzmzS2xXMeOling6KOT62/g5Zet51CrFrz7rifeLGhyItEupZ6kHj1aNTVVVUS1dm2biBo8uHTP5VxFEfn/OjW18AnqfftUFyxQHTZM9aqrVBs3zp3cbtpUtXt31eHDVRcvtnMro61bVa+7zt7zmWeqZmbGu0XlhkImqZM71caYMQd+06peHV54wb5pOZeMVG2YJaeHMXWq1TsBaNLEElR26mS9jOOPT/wexvz5tsfp228t7frf/2451pJEYak2kjtApKVZLYf8UlN9As+5HKqwbFnegJGZafc1bJg7JNWxo6UnT5QMpqrw7LPw179azfAxY2yvU5LxAFGQKlWiT8qJeH4b5wqiCt9/nxsspk3L/aJ12GHWw8gJGCefXDEXeWzeDL162dLVLl1sVVfjxvFuVVwUFiCSpx8VTUpK9B5EQatAnHP2BerII+1yww12bMWK3B7GtGlWdhcs68Bvf5sbMNq0if/wzRdf2N6mzEyr4VAJ6jaEJbkDREGrPQYNil+bnEtEaWm5S2YBVq/OGzDeftuOH3yw5ZHKWSV1yinlFzD27bPl6/37Q7Nmtnz1tNPK57UTVHIPMYGNOw4YYPWkU1IsOPgEtXOxtWZN3oDxzTd2vG5d6NAhd9I7PT2cTanr19sS9nfftaSGzz9v+0Kcz0E45yqYH3+E6dNzA8bChXa8Th34zW9yA8avfw01a5bttaZPt6I+WVnw2GO2zyHRV17FkAcI51zFlpWVN2DMn2/Ha9WygJEzh9G+ffFzIWVn24jAgw9aHY3XX7e8Si4PDxDOucSyYYPNEeQEjLlzbfVUzZo2b5ATME4/HWrXtsdEDhcfcYTNdyxebEPGw4dDvXrxfU8VlAcI51xi27QJPv00d2ntV1/ZpHONGtCuHTRoYOWAd+3K+7hevWyvgw8pFShuNalFpIuILBGRZSLSv5DzuomIikh6xLG7g8ctEZHzw2ync66Cq18fLrnE6nJnZMDGjTbh3LevFfOZMOHA4ADw3/96cCiD0NaXiUhVYBhwLpAJzBaRiaq6KN959YDbgFkRx44HugMnAEcAk0XkGFXNDqu9zrkEcsghlkgvJ5leQZteV60q33ZVMmH2INoBy1R1uaruBsYCXaOc9xAwBNgZcawrMFZVd6nq98Cy4Pmcc+5ABW1u9U2vZRJmgGgGrI64nRkc209E2gItVPWdkj42eHxvEckQkYysrKzYtNo5l3gGDbIlspF802uZhRkgog387e8DikgVYChwR0kfu/+A6ghVTVfV9EaNGpW6oc65BNejhxX3Sk21OYfUVLvtm17LJMw97plAi4jbzYE1EbfrAScCU8UmkQ4HJorIpcV4rHPO5dWjhweEGAuzBzEbaCUiLUWkBjbpPDHnTlXdoqoNVTVNVdOAmcClqpoRnNddRGqKSEugFfBFiG11zjmXT2g9CFXdKyK3Ah8AVYFRqrpQRAZiFYwmFvLYhSLyBrAI2Av08RVMzjlXvnyjnHPOJbG4bZRzzjmXuDxAOOeci6rSDDGJSBYQpTxcsTUE1seoOYki2d5zsr1f8PecLMrynlNVNeo+gUoTIMpKRDIKGoerrJLtPSfb+wV/z8kirPfsQ0zOOeei8gDhnHMuKg8QuUbEuwFxkGzvOdneL/h7ThahvGefg3DOOReV9yCcc85F5QHCOedcVEkfIERklIisE5EF8W5LeRCRFiIyRUQWi8hCEekb7zaFTURqicgXIjIveM8PxrtN5UVEqorIVyKSv+ZKpSQiK0TkaxGZKyJJkXtHRA4VkXEi8k3wd316zJ472ecgRORMYBvwsqqeGO/2hE1EmgJNVfXLoNzrHOCy/KVgKxOxfPIHqeo2EakOfAr0VdWZcW5a6ETkf4B04GBVvTje7QmbiKwA0lU1aTbKichLwCeqOjLInF1HVTfH4rmTvgehqtOBjfFuR3lR1bWq+mVwfSuwmCjV+ioTNduCm9WDS6X/ZiQizYGLgJHxbosLh4gcDJwJPA+gqrtjFRzAA0RSE5E0oC0wK74tCV8w1DIXWAd8qKqV/j0DjwN3Afvi3ZBypMB/RWSOiPSOd2PKwZFAFvBCMJQ4UkQOitWTe4BIUiJSF3gT+Kuq/hzv9oRNVbNVtQ1WnbCdiFTq4UQRuRhYp6pz4t2WctZBVU8BLgD6BEPIlVk14BRguKq2BX4B+sfqyT1AJKFgHP5NYIyq/ife7SlPQfd7KtAlzk0JWwfg0mBMfixwtoiMjm+Twqeqa4Kf64C3gHbxbVHoMoHMiB7xOCxgxIQHiCQTTNg+DyxW1cfi3Z7yICKNROTQ4Hpt4Bzgm/i2KlyqereqNg/K+XYHPlbVP8S5WaESkYOChRcEwyznAZV6daKq/gisFpFjg0OdsUqcMRFaydFEISKvAZ2AhiKSCdyvqs/Ht1Wh6gBcC3wdjMkD3KOq78WxTWFrCrwkIlWxL0VvqGpSLPtMMk2At+w7ENWAV1X1/fg2qVz8BRgTrGBaDtwQqydO+mWuzjnnovMhJuecc1F5gHDOOReVBwjnnHNReYBwzjkXlQcI55xzUXmAcK4IIpIdZAfNucRsp6qIpCVLJmGXeJJ+H4RzxbAjSNPhXFLxHoRzpRTUHvhnUGviCxE5OjieKiIficj84GdKcLyJiLwV1KWYJyK/CZ6qqog8F9Sq+G+w2xsRuU1EFgXPMzZOb9MlMQ8QzhWtdr4hpt9H3PezqrYDnsKypxJcf1lVTwbGAE8Ex58Apqnqr7B8OQuD462AYap6ArAZuDI43h9oGzzPzWG9OecK4jupnSuCiGxT1bpRjq8AzlbV5UECxB9VtYGIrMeKMu0Jjq9V1YYikgU0V9VdEc+RhqUfbxXc7gdUV9WHReR9rJjVeGB8RE0L58qF9yCcKxst4HpB50SzK+J6NrlzgxcBw4BTgTki4nOGrlx5gHCubH4f8fPz4PoMLIMqQA+sxCnAR8CfYX8Bo4MLelIRqQK0UNUpWNGfQ4EDejHOhcm/kThXtNoRmW8B3lfVnKWuNUVkFvZl6+rg2G3AKBH5G1btKye7Zl9ghIj8Cesp/BlYW8BrVgVGi8ghgABDY1lK0rni8DkI50opmINIV9X18W6Lc2HwISbnnHNReQ/COedcVN6DcM45F5UHCOecc1F5gHDOOReVBwjnnHNReYBwzjkX1f8DwrMmXZrNl8oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_metric(dfhistory, metric):\n",
    "    train_metrics = dfhistory[metric]\n",
    "    val_metrics = dfhistory['val_'+metric]\n",
    "    epochs = range(1, len(train_metrics) + 1)\n",
    "    plt.plot(epochs, train_metrics, 'bo--')\n",
    "    plt.plot(epochs, val_metrics, 'ro-')\n",
    "    plt.title('Training and validation '+ metric)\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(metric)\n",
    "    plt.legend([\"train_\"+metric, 'val_'+metric])\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# 观察损失和准确率的变化\n",
    "plot_metric(dfhistory,\"loss\")\n",
    "plot_metric(dfhistory,\"auc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:06:08.528336Z",
     "start_time": "2020-10-18T07:06:08.514360Z"
    },
    "run_control": {
     "marked": true
    }
   },
   "outputs": [],
   "source": [
    "# 预测\n",
    "y_pred_probs = net(torch.tensor(test_set.values).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:06:09.285781Z",
     "start_time": "2020-10-18T07:06:09.265833Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:06:12.382915Z",
     "start_time": "2020-10-18T07:06:12.363923Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的保存与使用\n",
    "torch.save(net.state_dict(), './model/net_parameter.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:06:17.090816Z",
     "start_time": "2020-10-18T07:06:17.081875Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[256, 128, 64]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden_units_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:06:22.108231Z",
     "start_time": "2020-10-18T07:06:22.091272Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net_clone = PNN(feature_info, hidden_units_copy)\n",
    "net_clone.load_state_dict(torch.load('./model/net_parameter.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:06:37.651779Z",
     "start_time": "2020-10-18T07:06:37.634824Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred_probs = net_clone(torch.tensor(test_set.values).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-18T07:06:46.451385Z",
     "start_time": "2020-10-18T07:06:46.441370Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
